From 97b77eec77896be7bf35c555b3ee40a607b36fe4 Mon Sep 17 00:00:00 2001 From: aneesh-n <99904+aneesh-n@users.noreply.github.com> Date: Mon, 2 Sep 2024 03:27:34 -0600 Subject: [PATCH] Refactor smb and local to use common helper --- internal/backend/local/local.go | 283 +++-------------------- internal/backend/smb/smb.go | 287 ++++-------------------- internal/backend/util/file_helper.go | 321 +++++++++++++++++++++++++++ 3 files changed, 395 insertions(+), 496 deletions(-) create mode 100644 internal/backend/util/file_helper.go diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index ee87ae5d6..8640e2a1d 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -2,12 +2,10 @@ package local import ( "context" - "fmt" "hash" "io" "os" "path/filepath" - "syscall" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" @@ -15,10 +13,6 @@ import ( "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/fs" - - "github.com/cenkalti/backoff/v4" ) // Local is a backend in a local directory. @@ -31,19 +25,13 @@ type Local struct { // ensure statically that *Local implements backend.Backend. var _ backend.Backend = &Local{} -var errTooShort = fmt.Errorf("file is too short") - func NewFactory() location.Factory { return location.NewLimitedBackendFactory("local", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open)) } func open(cfg Config) (*Local, error) { l := layout.NewDefaultLayout(cfg.Path, filepath.Join) - - fi, err := os.Stat(l.Filename(backend.Handle{Type: backend.ConfigFile})) - m := util.DeriveModesFromFileInfo(fi, err) - debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir) - + m := util.DeriveModesFromStat(l, os.Stat) return &Local{ Config: cfg, Layout: l, @@ -61,26 +49,14 @@ func Open(_ context.Context, cfg Config) (*Local, error) { // backend at dir. Afterwards a new config blob should be created. func Create(_ context.Context, cfg Config) (*Local, error) { debug.Log("create local backend at %v", cfg.Path) - be, err := open(cfg) if err != nil { return nil, err } - - // test if config file already exists - _, err = os.Lstat(be.Filename(backend.Handle{Type: backend.ConfigFile})) - if err == nil { - return nil, errors.New("config file already exists") + err = util.Create(be.Filename(backend.Handle{Type: backend.ConfigFile}), be.Modes.Dir, be.Paths(), os.Lstat, os.MkdirAll) + if err != nil { + return nil, err } - - // create paths for data and refs - for _, d := range be.Paths() { - err := os.MkdirAll(d, be.Modes.Dir) - if err != nil { - return nil, errors.WithStack(err) - } - } - return be, nil } @@ -100,106 +76,38 @@ func (b *Local) HasAtomicReplace() bool { // IsNotExist returns true if the error is caused by a non existing file. func (b *Local) IsNotExist(err error) bool { - return errors.Is(err, os.ErrNotExist) + return util.IsNotExist(err) } func (b *Local) IsPermanentError(err error) bool { - return b.IsNotExist(err) || errors.Is(err, errTooShort) || errors.Is(err, os.ErrPermission) + return util.IsPermanentError(err) } // Save stores data in the backend at the handle. func (b *Local) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) (err error) { - finalname := b.Filename(h) - dir := filepath.Dir(finalname) - - defer func() { - // Mark non-retriable errors as such - if errors.Is(err, syscall.ENOSPC) || os.IsPermission(err) { - err = backoff.Permanent(err) - } - }() - + fileName := b.Filename(h) // Create new file with a temporary name. - tmpname := filepath.Base(finalname) + "-tmp-" - f, err := tempFile(dir, tmpname) + tmpFilename := filepath.Base(fileName) + "-tmp-" - if b.IsNotExist(err) { - debug.Log("error %v: creating dir", err) - - // error is caused by a missing directory, try to create it - mkdirErr := os.MkdirAll(dir, b.Modes.Dir) - if mkdirErr != nil { - debug.Log("error creating dir %v: %v", dir, mkdirErr) - } else { - // try again - f, err = tempFile(dir, tmpname) - } + saveOptions := util.SaveOptions{ + OpenTempFile: func(dir, name string) (util.File, error) { + return tempFile(dir, name) + }, + MkDir: func(dir string) error { + return os.MkdirAll(dir, b.Modes.Dir) + }, + Remove: os.Remove, + IsMacENOTTY: isMacENOTTY, + Rename: os.Rename, + FsyncDir: fsyncDir, + SetFileReadonly: func(name string) error { + return os.Chmod(name, b.Modes.File) + }, + DirMode: b.Modes.Dir, + FileMode: b.Modes.File, } - if err != nil { - return errors.WithStack(err) - } - - defer func(f *os.File) { - if err != nil { - _ = f.Close() // Double Close is harmless. - // Remove after Rename is harmless: we embed the final name in the - // temporary's name and no other goroutine will get the same data to - // Save, so the temporary name should never be reused by another - // goroutine. - _ = os.Remove(f.Name()) - } - }(f) - - // preallocate disk space - if size := rd.Length(); size > 0 { - if err := fs.PreallocateFile(f, size); err != nil { - debug.Log("Failed to preallocate %v with size %v: %v", finalname, size, err) - } - } - - // save data, then sync - wbytes, err := io.Copy(f, rd) - if err != nil { - return errors.WithStack(err) - } - // sanity check - if wbytes != rd.Length() { - return errors.Errorf("wrote %d bytes instead of the expected %d bytes", wbytes, rd.Length()) - } - - // Ignore error if filesystem does not support fsync. - err = f.Sync() - syncNotSup := err != nil && (errors.Is(err, syscall.ENOTSUP) || isMacENOTTY(err)) - if err != nil && !syncNotSup { - return errors.WithStack(err) - } - - // Close, then rename. Windows doesn't like the reverse order. - if err = f.Close(); err != nil { - return errors.WithStack(err) - } - if err = os.Rename(f.Name(), finalname); err != nil { - return errors.WithStack(err) - } - - // Now sync the directory to commit the Rename. - if !syncNotSup { - err = fsyncDir(dir) - if err != nil { - return errors.WithStack(err) - } - } - - // try to mark file as read-only to avoid accidental modifications - // ignore if the operation fails as some filesystems don't allow the chmod call - // e.g. exfat and network file systems with certain mount options - err = setFileReadonly(finalname, b.Modes.File) - if err != nil && !os.IsPermission(err) { - return errors.WithStack(err) - } - - return nil + return util.SaveWithOptions(fileName, tmpFilename, rd, saveOptions) } var tempFile = os.CreateTemp // Overridden by test. @@ -211,153 +119,30 @@ func (b *Local) Load(ctx context.Context, h backend.Handle, length int, offset i } func (b *Local) openReader(_ context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { - f, err := os.Open(b.Filename(h)) - if err != nil { - return nil, err + openFile := func(name string) (util.File, error) { + return os.Open(name) } - - fi, err := f.Stat() - if err != nil { - _ = f.Close() - return nil, err - } - - size := fi.Size() - if size < offset+int64(length) { - _ = f.Close() - return nil, errTooShort - } - - if offset > 0 { - _, err = f.Seek(offset, 0) - if err != nil { - _ = f.Close() - return nil, err - } - } - - if length > 0 { - return util.LimitReadCloser(f, int64(length)), nil - } - - return f, nil + return util.OpenReader(openFile, b.Filename(h), length, offset) } // Stat returns information about a blob. func (b *Local) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) { - fi, err := os.Stat(b.Filename(h)) - if err != nil { - return backend.FileInfo{}, errors.WithStack(err) - } - - return backend.FileInfo{Size: fi.Size(), Name: h.Name}, nil + return util.Stat(os.Stat, b.Filename(h), h.Name) } // Remove removes the blob with the given name and type. func (b *Local) Remove(_ context.Context, h backend.Handle) error { - fn := b.Filename(h) - - // reset read-only flag - err := os.Chmod(fn, 0666) - if err != nil && !os.IsPermission(err) { - return errors.WithStack(err) - } - - return os.Remove(fn) + return util.Remove(b.Filename(h), os.Chmod) } // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. func (b *Local) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) (err error) { + openFunc := func(name string) (util.File, error) { + return os.Open(name) + } basedir, subdirs := b.Basedir(t) - if subdirs { - err = visitDirs(ctx, basedir, fn) - } else { - err = visitFiles(ctx, basedir, fn, false) - } - - if b.IsNotExist(err) { - debug.Log("ignoring non-existing directory") - return nil - } - - return err -} - -// The following two functions are like filepath.Walk, but visit only one or -// two levels of directory structure (including dir itself as the first level). -// Also, visitDirs assumes it sees a directory full of directories, while -// visitFiles wants a directory full or regular files. -func visitDirs(ctx context.Context, dir string, fn func(backend.FileInfo) error) error { - d, err := os.Open(dir) - if err != nil { - return err - } - - sub, err := d.Readdirnames(-1) - if err != nil { - // ignore subsequent errors - _ = d.Close() - return err - } - - err = d.Close() - if err != nil { - return err - } - - for _, f := range sub { - err = visitFiles(ctx, filepath.Join(dir, f), fn, true) - if err != nil { - return err - } - } - return ctx.Err() -} - -func visitFiles(ctx context.Context, dir string, fn func(backend.FileInfo) error, ignoreNotADirectory bool) error { - d, err := os.Open(dir) - if err != nil { - return err - } - - if ignoreNotADirectory { - fi, err := d.Stat() - if err != nil || !fi.IsDir() { - // ignore subsequent errors - _ = d.Close() - return err - } - } - - sub, err := d.Readdir(-1) - if err != nil { - // ignore subsequent errors - _ = d.Close() - return err - } - - err = d.Close() - if err != nil { - return err - } - - for _, fi := range sub { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - err := fn(backend.FileInfo{ - Name: fi.Name(), - Size: fi.Size(), - }) - if err != nil { - return err - } - } - return nil + return util.List(ctx, basedir, subdirs, openFunc, t, fn) } // Delete removes the repository and all files. diff --git a/internal/backend/smb/smb.go b/internal/backend/smb/smb.go index b5f00751f..87a7332f0 100644 --- a/internal/backend/smb/smb.go +++ b/internal/backend/smb/smb.go @@ -10,18 +10,14 @@ import ( "path" "path/filepath" "sync" - "syscall" "time" - "github.com/cenkalti/backoff/v4" - "github.com/hirochachacha/go-smb2" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/limiter" "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/errors" ) // Parts of this code have been adapted from Rclone (https://github.com/rclone) @@ -52,16 +48,14 @@ type SMB struct { pool []*conn drain *time.Timer // used to drain the pool when we stop using the connections - layout.Layout Config + layout.Layout util.Modes } // ensure statically that *SMB implements backend.Backend. var _ backend.Backend = &SMB{} -var errTooShort = errors.New("file is too short") - func NewFactory() location.Factory { return location.NewLimitedBackendFactory("smb", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open)) } @@ -87,12 +81,7 @@ func open(cfg Config) (*SMB, error) { } defer b.putConnection(cn) - stat, err := cn.smbShare.Stat(l.Filename(backend.Handle{Type: backend.ConfigFile})) - m := util.DeriveModesFromFileInfo(stat, err) - debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir) - - b.Modes = m - + b.Modes = util.DeriveModesFromStat(l, cn.smbShare.Stat) return b, nil } @@ -105,32 +94,20 @@ func Open(_ context.Context, cfg Config) (*SMB, error) { // Create creates all the necessary files and directories for a new SMB backend. func Create(_ context.Context, cfg Config) (*SMB, error) { debug.Log("create smb backend at %v (share %q)", cfg.Path, cfg.ShareName) - b, err := open(cfg) if err != nil { return nil, err } - cn, err := b.getConnection(cfg.ShareName) if err != nil { return b, err } defer b.putConnection(cn) - // test if config file already exists - _, err = cn.smbShare.Lstat(b.Filename(backend.Handle{Type: backend.ConfigFile})) - if err == nil { - return nil, errors.New("config file already exists") + err = util.Create(b.Filename(backend.Handle{Type: backend.ConfigFile}), b.Modes.Dir, b.Paths(), cn.smbShare.Lstat, cn.smbShare.MkdirAll) + if err != nil { + return nil, err } - - // create paths for data and refs - for _, d := range b.Paths() { - err := cn.smbShare.MkdirAll(d, b.Modes.Dir) - if err != nil { - return nil, errors.WithStack(err) - } - } - return b, nil } @@ -150,31 +127,15 @@ func (b *SMB) HasAtomicReplace() bool { // IsNotExist returns true if the error is caused by a non existing file. func (b *SMB) IsNotExist(err error) bool { - return errors.Is(err, os.ErrNotExist) -} - -// Join combines path components with slashes. -func (b *SMB) Join(p ...string) string { - return path.Join(p...) + return util.IsNotExist(err) } func (b *SMB) IsPermanentError(err error) bool { - return b.IsNotExist(err) || errors.Is(err, errTooShort) || errors.Is(err, os.ErrPermission) + return util.IsPermanentError(err) } // Save stores data in the backend at the handle. func (b *SMB) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) (err error) { - filename := b.Filename(h) - tmpFilename := filename + "-restic-temp-" + tempSuffix() - dir := filepath.Dir(tmpFilename) - - defer func() { - // Mark non-retriable errors as such - if errors.Is(err, syscall.ENOSPC) || os.IsPermission(err) { - err = backoff.Permanent(err) - } - }() - b.addSession() // Show session in use defer b.removeSession() @@ -185,73 +146,32 @@ func (b *SMB) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) } defer b.putConnection(cn) - var f *smb2.File - // create new file - f, err = cn.smbShare.OpenFile(tmpFilename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + fileName := b.Filename(h) + tmpFilename := fileName + "-restic-temp-" + tempSuffix() - if b.IsNotExist(err) { - debug.Log("error %v: creating dir", err) - - // error is caused by a missing directory, try to create it - mkdirErr := cn.smbShare.MkdirAll(dir, b.Modes.Dir) - if mkdirErr != nil { - debug.Log("error creating dir %v: %v", dir, mkdirErr) - } else { - // try again - f, err = cn.smbShare.OpenFile(tmpFilename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) - } + saveOptions := util.SaveOptions{ + OpenTempFile: func(dir, name string) (util.File, error) { + return cn.smbShare.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + }, + MkDir: func(dir string) error { + return cn.smbShare.MkdirAll(dir, b.Modes.Dir) + }, + Remove: cn.smbShare.Remove, + IsMacENOTTY: func(error) bool { + return false + }, + Rename: cn.smbShare.Rename, + FsyncDir: func(dir string) error { + return nil + }, + SetFileReadonly: func(name string) error { + return cn.setFileReadonly(name, b.Modes.File) + }, + DirMode: b.Modes.Dir, + FileMode: b.Modes.File, } - if err != nil { - return errors.WithStack(err) - } - - defer func(f *smb2.File) { - if err != nil { - _ = f.Close() // Double Close is harmless. - // Remove after Rename is harmless: we embed the final name in the - // temporary's name and no other goroutine will get the same data to - // Save, so the temporary name should never be reused by another - // goroutine. - _ = cn.smbShare.Remove(f.Name()) - } - }(f) - - // save data, then sync - wbytes, err := io.Copy(f, rd) - if err != nil { - return errors.WithStack(err) - } - // sanity check - if wbytes != rd.Length() { - return errors.Errorf("wrote %d bytes instead of the expected %d bytes", wbytes, rd.Length()) - } - - // Ignore error if filesystem does not support fsync. - // In this case the sync call is on the smb client's file. - err = f.Sync() - syncNotSup := err != nil && (errors.Is(err, syscall.ENOTSUP)) - if err != nil && !syncNotSup { - return errors.WithStack(err) - } - - // Close, then rename. Windows doesn't like the reverse order. - if err = f.Close(); err != nil { - return errors.WithStack(err) - } - if err = cn.smbShare.Rename(f.Name(), filename); err != nil { - return errors.WithStack(err) - } - - // try to mark file as read-only to avoid accidental modifications - // ignore if the operation fails as some filesystems don't allow the chmod call - // e.g. exfat and network file systems with certain mount options - err = cn.setFileReadonly(filename, b.Modes.File) - if err != nil && !os.IsPermission(err) { - return errors.WithStack(err) - } - - return nil + return util.SaveWithOptions(fileName, tmpFilename, rd, saveOptions) } // set file to readonly @@ -273,37 +193,10 @@ func (b *SMB) openReader(_ context.Context, h backend.Handle, length int, offset return nil, err } defer b.putConnection(cn) - - f, err := cn.smbShare.Open(b.Filename(h)) - if err != nil { - return nil, err + openFile := func(name string) (util.File, error) { + return cn.smbShare.Open(name) } - - fi, err := f.Stat() - if err != nil { - _ = f.Close() - return nil, err - } - - size := fi.Size() - if size < offset+int64(length) { - _ = f.Close() - return nil, errTooShort - } - - if offset > 0 { - _, err = f.Seek(offset, 0) - if err != nil { - _ = f.Close() - return nil, err - } - } - - if length > 0 { - return util.LimitReadCloser(f, int64(length)), nil - } - - return f, nil + return util.OpenReader(openFile, b.Filename(h), length, offset) } // Stat returns information about a blob. @@ -313,32 +206,17 @@ func (b *SMB) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error return backend.FileInfo{}, err } defer b.putConnection(cn) - - fi, err := cn.smbShare.Stat(b.Filename(h)) - if err != nil { - return backend.FileInfo{}, errors.WithStack(err) - } - - return backend.FileInfo{Size: fi.Size(), Name: h.Name}, nil + return util.Stat(cn.smbShare.Stat, b.Filename(h), h.Name) } // Remove removes the blob with the given name and type. func (b *SMB) Remove(_ context.Context, h backend.Handle) error { - fn := b.Filename(h) - cn, err := b.getConnection(b.ShareName) if err != nil { return err } defer b.putConnection(cn) - - // reset read-only flag - err = cn.smbShare.Chmod(fn, 0666) - if err != nil && !os.IsPermission(err) { - return errors.WithStack(err) - } - - return cn.smbShare.Remove(fn) + return util.Remove(b.Filename(h), cn.smbShare.Chmod) } // List runs fn for each file in the backend which has the type t. When an @@ -349,96 +227,11 @@ func (b *SMB) List(ctx context.Context, t backend.FileType, fn func(backend.File return err } defer b.putConnection(cn) - + openFunc := func(name string) (util.File, error) { + return cn.smbShare.Open(name) + } basedir, subdirs := b.Basedir(t) - if subdirs { - err = b.visitDirs(ctx, cn, basedir, fn) - } else { - err = b.visitFiles(ctx, cn, basedir, fn, false) - } - - if b.IsNotExist(err) { - debug.Log("ignoring non-existing directory") - return nil - } - - return err -} - -// The following two functions are like filepath.Walk, but visit only one or -// two levels of directory structure (including dir itself as the first level). -// Also, visitDirs assumes it sees a directory full of directories, while -// visitFiles wants a directory full or regular files. -func (b *SMB) visitDirs(ctx context.Context, cn *conn, dir string, fn func(backend.FileInfo) error) error { - d, err := cn.smbShare.Open(dir) - if err != nil { - return err - } - - sub, err := d.Readdirnames(-1) - if err != nil { - // ignore subsequent errors - _ = d.Close() - return err - } - - err = d.Close() - if err != nil { - return err - } - - for _, f := range sub { - err = b.visitFiles(ctx, cn, filepath.Join(dir, f), fn, true) - if err != nil { - return err - } - } - return ctx.Err() -} - -func (b *SMB) visitFiles(ctx context.Context, cn *conn, dir string, fn func(backend.FileInfo) error, ignoreNotADirectory bool) error { - d, err := cn.smbShare.Open(dir) - if err != nil { - return err - } - - if ignoreNotADirectory { - fi, err := d.Stat() - if err != nil || !fi.IsDir() { - // ignore subsequent errors - _ = d.Close() - return err - } - } - - sub, err := d.Readdir(-1) - if err != nil { - // ignore subsequent errors - _ = d.Close() - return err - } - - err = d.Close() - if err != nil { - return err - } - - for _, fi := range sub { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - err := fn(backend.FileInfo{ - Name: fi.Name(), - Size: fi.Size(), - }) - if err != nil { - return err - } - } - return nil + return util.List(ctx, basedir, subdirs, openFunc, t, fn) } // Delete removes the repository and all files. @@ -448,7 +241,7 @@ func (b *SMB) Delete(_ context.Context) error { return err } defer b.putConnection(cn) - return cn.smbShare.RemoveAll(b.Join(b.ShareName, b.Path)) + return cn.smbShare.RemoveAll(path.Join(b.ShareName, b.Path)) } // Close closes all open files. diff --git a/internal/backend/util/file_helper.go b/internal/backend/util/file_helper.go new file mode 100644 index 000000000..37e307e62 --- /dev/null +++ b/internal/backend/util/file_helper.go @@ -0,0 +1,321 @@ +package util + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "syscall" + + "github.com/cenkalti/backoff/v4" + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/layout" + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/fs" +) + +// File is common interface for os.File and smb.File +type File interface { + Close() error + Name() string + Read(p []byte) (n int, err error) + Readdir(count int) ([]os.FileInfo, error) + Readdirnames(n int) ([]string, error) + Seek(offset int64, whence int) (int64, error) + Stat() (os.FileInfo, error) + Sync() error + Write(p []byte) (n int, err error) +} + +var errTooShort = fmt.Errorf("file is too short") + +func DeriveModesFromStat(l layout.Layout, statFn func(string) (os.FileInfo, error)) Modes { + fi, err := statFn(l.Filename(backend.Handle{Type: backend.ConfigFile})) + m := DeriveModesFromFileInfo(fi, err) + debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir) + return m +} + +// Create creates all the necessary files and directories for a new local +// backend at dir. Afterwards a new config blob should be created. +func Create(fileName string, dirMode os.FileMode, paths []string, lstatFn func(string) (os.FileInfo, error), MkdirAllFn func(string, os.FileMode) error) error { + // test if config file already exists + _, err := lstatFn(fileName) + if err == nil { + return errors.New("config file already exists") + } + // create paths for data and refs + for _, d := range paths { + err := MkdirAllFn(d, dirMode) + if err != nil { + return errors.WithStack(err) + } + } + return nil +} + +// SaveOptions contains options for saving files. +type SaveOptions struct { + OpenTempFile func(dir, name string) (File, error) + MkDir func(dir string) error + Remove func(name string) error + IsMacENOTTY func(error) bool + Rename func(oldname, newname string) error + FsyncDir func(dir string) error + SetFileReadonly func(name string) error + DirMode os.FileMode + FileMode os.FileMode +} + +// SaveWithOptions stores data in the backend at the handle using the provided options. +func SaveWithOptions(fileName string, tmpFilename string, rd backend.RewindReader, options SaveOptions) (err error) { + dir := filepath.Dir(fileName) + + defer func() { + // Mark non-retriable errors as such + if errors.Is(err, syscall.ENOSPC) || os.IsPermission(err) { + err = backoff.Permanent(err) + } + }() + + f, err := options.OpenTempFile(dir, tmpFilename) + + if IsNotExist(err) { + debug.Log("error %v: creating dir", err) + + // error is caused by a missing directory, try to create it + mkdirErr := options.MkDir(dir) + if mkdirErr != nil { + debug.Log("error creating dir %v: %v", dir, mkdirErr) + } else { + // try again + f, err = options.OpenTempFile(dir, tmpFilename) + } + } + + if err != nil { + return errors.WithStack(err) + } + + defer func(f File) { + if err != nil { + _ = f.Close() // Double Close is harmless. + // Remove after Rename is harmless: we embed the final name in the + // temporary's name and no other goroutine will get the same data to + // Save, so the temporary name should never be reused by another + // goroutine. + _ = options.Remove(f.Name()) + } + }(f) + + if f, ok := f.(*os.File); ok { + // preallocate disk space only for os.File + if size := rd.Length(); size > 0 { + if err := fs.PreallocateFile(f, size); err != nil { + debug.Log("Failed to preallocate %v with size %v: %v", fileName, size, err) + } + } + } + + // save data, then sync + wbytes, err := io.Copy(f, rd) + if err != nil { + return errors.WithStack(err) + } + // sanity check + if wbytes != rd.Length() { + return errors.Errorf("wrote %d bytes instead of the expected %d bytes", wbytes, rd.Length()) + } + + // Ignore error if filesystem does not support fsync. + err = f.Sync() + syncNotSup := err != nil && (errors.Is(err, syscall.ENOTSUP) || options.IsMacENOTTY(err)) + if err != nil && !syncNotSup { + return errors.WithStack(err) + } + + // Close, then rename. Windows doesn't like the reverse order. + if err = f.Close(); err != nil { + return errors.WithStack(err) + } + if err = options.Rename(f.Name(), fileName); err != nil { + return errors.WithStack(err) + } + + // Now sync the directory to commit the Rename. + if !syncNotSup { + err = options.FsyncDir(dir) + if err != nil { + return errors.WithStack(err) + } + } + + // try to mark file as read-only to avoid accidental modifications + // ignore if the operation fails as some filesystems don't allow the chmod call + // e.g. exfat and network file systems with certain mount options + err = options.SetFileReadonly(fileName) + if err != nil && !os.IsPermission(err) { + return errors.WithStack(err) + } + + return nil +} + +func OpenReader(openFile func(string) (File, error), fileName string, length int, offset int64) (io.ReadCloser, error) { + f, err := openFile(fileName) + if err != nil { + return nil, err + } + fi, err := f.Stat() + if err != nil { + _ = f.Close() + return nil, err + } + + size := fi.Size() + if size < offset+int64(length) { + _ = f.Close() + return nil, errTooShort + } + + if offset > 0 { + _, err = f.Seek(offset, 0) + if err != nil { + _ = f.Close() + return nil, err + } + } + + if length > 0 { + return LimitReadCloser(f, int64(length)), nil + } + + return f, nil +} + +// Stat returns information about a blob. +func Stat(statFn func(string) (os.FileInfo, error), fileName, handleName string) (backend.FileInfo, error) { + fi, err := statFn(fileName) + if err != nil { + return backend.FileInfo{}, errors.WithStack(err) + } + return backend.FileInfo{Size: fi.Size(), Name: handleName}, nil +} + +// Remove removes the blob with the given name and type. +func Remove(filename string, chmodfn func(string, os.FileMode) error) error { + // reset read-only flag + err := chmodfn(filename, 0666) + if err != nil && !os.IsPermission(err) { + return errors.WithStack(err) + } + + return os.Remove(filename) +} + +// List runs fn for each file in the backend which has the type t. When an +// error occurs (or fn returns an error), List stops and returns it. +func List(ctx context.Context, basedir string, subdirs bool, openFunc func(name string) (File, error), t backend.FileType, fn func(backend.FileInfo) error) (err error) { + if subdirs { + err = visitDirs(ctx, openFunc, basedir, fn) + } else { + err = visitFiles(ctx, openFunc, basedir, fn, false) + } + + if IsNotExist(err) { + debug.Log("ignoring non-existing directory") + return nil + } + + return err +} + +// The following two functions are like filepath.Walk, but visit only one or +// two levels of directory structure (including dir itself as the first level). +// Also, visitDirs assumes it sees a directory full of directories, while +// visitFiles wants a directory full or regular files. +// visitDirs visits directories +func visitDirs(ctx context.Context, openDir func(string) (File, error), dir string, fn func(backend.FileInfo) error) error { + d, err := openDir(dir) + if err != nil { + return err + } + + sub, err := d.Readdirnames(-1) + if err != nil { + // ignore subsequent errors + _ = d.Close() + return err + } + + err = d.Close() + if err != nil { + return err + } + + for _, f := range sub { + err = visitFiles(ctx, openDir, filepath.Join(dir, f), fn, true) + if err != nil { + return err + } + } + return ctx.Err() +} + +// visitFiles visits files +func visitFiles(ctx context.Context, openDir func(string) (File, error), dir string, fn func(backend.FileInfo) error, ignoreNotADirectory bool) error { + d, err := openDir(dir) + if err != nil { + return err + } + + if ignoreNotADirectory { + fi, err := d.Stat() + if err != nil || !fi.IsDir() { + // ignore subsequent errors + _ = d.Close() + return err + } + } + + sub, err := d.Readdir(-1) + if err != nil { + // ignore subsequent errors + _ = d.Close() + return err + } + + err = d.Close() + if err != nil { + return err + } + + for _, fi := range sub { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + err := fn(backend.FileInfo{ + Name: fi.Name(), + Size: fi.Size(), + }) + if err != nil { + return err + } + } + return nil +} + +// IsNotExist returns true if the error is caused by a non existing file. +func IsNotExist(err error) bool { + return errors.Is(err, os.ErrNotExist) +} + +// IsPermanentError checks if the error is permanent +func IsPermanentError(err error) bool { + return IsNotExist(err) || errors.Is(err, errTooShort) || errors.Is(err, os.ErrPermission) +}