diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index 55b6ee4b3..7521d92c9 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -242,21 +242,21 @@ func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s I case restic.NodeTypeDir: switch { case previous == nil: - arch.summary.Dirs.New++ + arch.summary.Dirs.incrementNewFiles(current) case previous.Equals(*current): - arch.summary.Dirs.Unchanged++ + arch.summary.Dirs.incrementUnchangedFiles(current) default: - arch.summary.Dirs.Changed++ + arch.summary.Dirs.incrementChangedFiles(current) } case restic.NodeTypeFile: switch { case previous == nil: - arch.summary.Files.New++ + arch.summary.Files.incrementNewFiles(current) case previous.Equals(*current): - arch.summary.Files.Unchanged++ + arch.summary.Files.incrementUnchangedFiles(current) default: - arch.summary.Files.Changed++ + arch.summary.Files.incrementChangedFiles(current) } } } @@ -320,17 +320,20 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me if err != nil { return futureNode{}, err } + pathnames := arch.preProcessPaths(dir, names) + sort.Strings(pathnames) - nodes := make([]futureNode, 0, len(names)) + nodes := make([]futureNode, 0, len(pathnames)) - for _, name := range names { + for _, pathname := range pathnames { // test if context has been cancelled if ctx.Err() != nil { debug.Log("context has been cancelled, aborting") return futureNode{}, ctx.Err() } - pathname := arch.FS.Join(dir, name) + name := getNameFromPathname(pathname) + pathname := arch.processPath(dir, pathname) oldNode := previous.Find(name) snItem := join(snPath, name) fn, excluded, err := arch.save(ctx, snItem, pathname, oldNode) @@ -343,7 +346,7 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me continue } - return futureNode{}, err + return futureNode{}, errors.Wrap(err, "error saving a target (file or directory)") } if excluded { @@ -456,7 +459,11 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous if err != nil { return futureNode{}, false, err } - + //In case of windows ADS files for checking include and excludes we use the main file which has the ADS files attached. + //For Unix, the main file is the same as there is no ADS. So targetMain is always the same as target. + //After checking the exclusion for actually processing the file, we use the full file name including ads portion if any. + targetMain := fs.SanitizeMainFileName(target) + abstargetMain := fs.SanitizeMainFileName(abstarget) filterError := func(err error) (futureNode, bool, error) { err = arch.error(abstarget, err) if err != nil { @@ -493,16 +500,21 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous }() // get file info and run remaining select functions that require file information - fi, err := meta.Stat() + fiMain, err := meta.Stat() if err != nil { debug.Log("lstat() for %v returned error: %v", target, err) // ignore if file disappeared since it was returned by readdir return filterError(filterNotExist(err)) } - if !arch.Select(abstarget, fi, arch.FS) { + if !arch.Select(abstargetMain, fiMain, arch.FS) { debug.Log("%v is excluded", target) return futureNode{}, true, nil } + var fi *fs.ExtendedFileInfo + fi, shouldReturn, fn, excluded, err := arch.processTargets(target, targetMain, abstarget, *fiMain) + if shouldReturn { + return fn, excluded, err + } switch { case fi.Mode.IsRegular(): @@ -694,11 +706,6 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, } return futureNode{}, 0, err } - - if err != nil { - return futureNode{}, 0, err - } - if !excluded { nodes = append(nodes, fn) } @@ -762,13 +769,9 @@ func (arch *Archiver) dirPathToNode(snPath, target string) (node *restic.Node, e func resolveRelativeTargets(filesys fs.FS, targets []string) ([]string, error) { debug.Log("targets before resolving: %v", targets) result := make([]string, 0, len(targets)) + preProcessTargets(filesys, &targets) for _, target := range targets { - if target != "" && filesys.VolumeName(target) == target { - // special case to allow users to also specify a volume name "C:" instead of a path "C:\" - target = target + filesys.Separator() - } else { - target = filesys.Clean(target) - } + target = processTarget(filesys, target) pc, _ := pathComponents(filesys, target, false) if len(pc) > 0 { result = append(result, target) diff --git a/internal/filter/filter.go b/internal/filter/filter.go index 473f1f4cb..812e5bd5a 100644 --- a/internal/filter/filter.go +++ b/internal/filter/filter.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/fs" ) // ErrBadString is returned when Match is called with the empty string as the @@ -200,9 +201,9 @@ func match(pattern Pattern, strs []string) (matched bool, err error) { for i := len(pattern.parts) - 1; i >= 0; i-- { var ok bool if pattern.parts[i].isSimple { - ok = pattern.parts[i].pattern == strs[offset+i] + ok = pattern.parts[i].pattern == fs.SanitizeMainFileName(strs[offset+i]) } else { - ok, err = filepath.Match(pattern.parts[i].pattern, strs[offset+i]) + ok, err = filepath.Match(pattern.parts[i].pattern, fs.SanitizeMainFileName(strs[offset+i])) if err != nil { return false, errors.Wrap(err, "Match") } diff --git a/internal/fs/file_unix.go b/internal/fs/file_unix.go index 4e7765c30..6f5e1f117 100644 --- a/internal/fs/file_unix.go +++ b/internal/fs/file_unix.go @@ -48,3 +48,9 @@ func chmod(name string, mode os.FileMode) error { return err } + +// SanitizeMainFileName will only keep the main file and remove the secondary file. +func SanitizeMainFileName(str string) string { + // no-op - In case of non-windows there is no secondary file + return str +} diff --git a/internal/fs/file_windows.go b/internal/fs/file_windows.go index d7aabf360..15f9dba5a 100644 --- a/internal/fs/file_windows.go +++ b/internal/fs/file_windows.go @@ -134,3 +134,11 @@ func openHandleForEA(nodeType restic.NodeType, path string, writeAccess bool) (h } return handle, err } + +// SanitizeMainFileName will only keep the main file and remove the secondary file like ADS from the name. +func SanitizeMainFileName(str string) string { + // The ADS is essentially a part of the main file. So for any functionality that + // needs to consider the main file, like filtering, we need to derive the main file name + // from the ADS name. + return restic.TrimAds(str) +} diff --git a/internal/restorer/fileswriter.go b/internal/restorer/fileswriter.go index d6f78f2d7..312f1ebad 100644 --- a/internal/restorer/fileswriter.go +++ b/internal/restorer/fileswriter.go @@ -4,11 +4,9 @@ import ( "fmt" "os" "sync" - "syscall" "github.com/cespare/xxhash/v2" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" ) @@ -61,27 +59,10 @@ func openFile(path string) (*os.File, error) { return f, nil } -func createFile(path string, createSize int64, sparse bool, allowRecursiveDelete bool) (*os.File, error) { - f, err := fs.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_NOFOLLOW, 0600) - if err != nil && fs.IsAccessDenied(err) { - // If file is readonly, clear the readonly flag by resetting the - // permissions of the file and try again - // as the metadata will be set again in the second pass and the - // readonly flag will be applied again if needed. - if err = fs.ResetPermissions(path); err != nil { - return nil, err - } - if f, err = fs.OpenFile(path, fs.O_WRONLY|fs.O_NOFOLLOW, 0600); err != nil { - return nil, err - } - } else if err != nil && (errors.Is(err, syscall.ELOOP) || errors.Is(err, syscall.EISDIR)) { - // symlink or directory, try to remove it later on - f = nil - } else if err != nil { - return nil, err - } +func postCreateFile(f *os.File, path string, createSize int64, allowRecursiveDelete, sparse bool) (*os.File, error) { var fi os.FileInfo + var err error if f != nil { // stat to check that we've opened a regular file fi, err = f.Stat() @@ -162,7 +143,7 @@ func ensureSize(f *os.File, fi os.FileInfo, createSize int64, sparse bool) (*os. return f, nil } -func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, createSize int64, sparse bool) error { +func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, createSize int64, fileInfo *fileInfo) error { bucket := &w.buckets[uint(xxhash.Sum64String(path))%uint(len(w.buckets))] acquireWriter := func() (*partialFile, error) { @@ -173,18 +154,12 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create bucket.files[path].users++ return wr, nil } - var f *os.File - var err error - if createSize >= 0 { - f, err = createFile(path, createSize, sparse, w.allowRecursiveDelete) - if err != nil { - return nil, err - } - } else if f, err = openFile(path); err != nil { + f, err := createOrOpenFile(path, createSize, fileInfo, w.allowRecursiveDelete) + if err != nil { return nil, err } - wr := &partialFile{File: f, users: 1, sparse: sparse} + wr := &partialFile{File: f, users: 1, sparse: fileInfo.sparse} bucket.files[path] = wr return wr, nil @@ -196,6 +171,8 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create if bucket.files[path].users == 1 { delete(bucket.files, path) + //Clean up for the path + CleanupPath(path) return wr.Close() } bucket.files[path].users-- diff --git a/internal/restorer/fileswriter_unix.go b/internal/restorer/fileswriter_unix.go new file mode 100644 index 000000000..554a4ab5a --- /dev/null +++ b/internal/restorer/fileswriter_unix.go @@ -0,0 +1,89 @@ +//go:build !windows +// +build !windows + +package restorer + +import ( + "os" + "syscall" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/fs" +) + +// OpenFile opens the file with create, truncate and write only options if +// createSize is specified greater than 0 i.e. if the file hasn't already +// been created. Otherwise it opens the file with only write only option. +func (fw *filesWriter) OpenFile(createSize int64, path string, fileInfo *fileInfo) (file *os.File, err error) { + return fw.openFile(createSize, path, fileInfo) +} + +// OpenFile opens the file with create, truncate and write only options if +// createSize is specified greater than 0 i.e. if the file hasn't already +// been created. Otherwise it opens the file with only write only option. +func (fw *filesWriter) openFile(createSize int64, path string, _ *fileInfo) (file *os.File, err error) { + if createSize >= 0 { + file, err = openFileWithCreate(path) + if fs.IsAccessDenied(err) { + // If file is readonly, clear the readonly flag by resetting the + // permissions of the file and try again + // as the metadata will be set again in the second pass and the + // readonly flag will be applied again if needed. + err = fs.ResetPermissions(path) + if err != nil { + return nil, err + } + file, err = openFileWithTruncWrite(path) + } + } else { + flags := os.O_WRONLY + file, err = os.OpenFile(path, flags, 0600) + } + return file, err +} + +// openFileWithCreate opens the file with os.O_CREATE flag along with os.O_TRUNC and os.O_WRONLY. +func openFileWithCreate(path string) (file *os.File, err error) { + flags := os.O_CREATE | os.O_TRUNC | os.O_WRONLY + return os.OpenFile(path, flags, 0600) +} + +// openFileWithTruncWrite opens the file without os.O_CREATE flag along with os.O_TRUNC and os.O_WRONLY. +func openFileWithTruncWrite(path string) (file *os.File, err error) { + flags := os.O_TRUNC | os.O_WRONLY + return os.OpenFile(path, flags, 0600) +} + +// CleanupPath performs clean up for the specified path. +func CleanupPath(_ string) { + // no-op +} + +func createOrOpenFile(path string, createSize int64, fileInfo *fileInfo, allowRecursiveDelete bool) (*os.File, error) { + if createSize >= 0 { + return createFile(path, createSize, fileInfo, allowRecursiveDelete) + } + return openFile(path) +} + +func createFile(path string, createSize int64, fileInfo *fileInfo, allowRecursiveDelete bool) (*os.File, error) { + f, err := fs.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_NOFOLLOW, 0600) + if err != nil && fs.IsAccessDenied(err) { + // If file is readonly, clear the readonly flag by resetting the + // permissions of the file and try again + // as the metadata will be set again in the second pass and the + // readonly flag will be applied again if needed. + if err = fs.ResetPermissions(path); err != nil { + return nil, err + } + if f, err = fs.OpenFile(path, fs.O_WRONLY|fs.O_NOFOLLOW, 0600); err != nil { + return nil, err + } + } else if err != nil && (errors.Is(err, syscall.ELOOP) || errors.Is(err, syscall.EISDIR)) { + // symlink or directory, try to remove it later on + f = nil + } else if err != nil { + return nil, err + } + return postCreateFile(f, path, createSize, allowRecursiveDelete, fileInfo.sparse) +} diff --git a/internal/restorer/fileswriter_windows.go b/internal/restorer/fileswriter_windows.go new file mode 100644 index 000000000..fccc93425 --- /dev/null +++ b/internal/restorer/fileswriter_windows.go @@ -0,0 +1,237 @@ +package restorer + +import ( + "encoding/json" + "errors" + "os" + "sync" + "syscall" + + "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/restic" +) + +// createOrOpenFile opens the file and handles the readonly attribute and ads related logic during file creation. +// Readonly files - if an existing file is detected as readonly we clear the flag because otherwise we cannot +// make changes to the file. The readonly attribute would be set again in the second pass when the attributes +// are set if the file version being restored has the readonly bit. +// ADS files need special handling - Each stream is treated as a separate file in restic. This method is called +// for the main file which has the streams and for each stream. +// If the ads stream calls this method first and the main file doesn't already exist, then creating the file +// for the streams causes the main file to automatically get created with 0 size. Hence we need to be careful +// while creating the main file. If we blindly create it with the os.O_CREATE option, it could overwrite the +// stream. However creating the stream with os.O_CREATE option does not overwrite the mainfile if it already +// exists. It will simply attach the new stream to the main file if the main file existed, otherwise it will +// create the 0 size main file. +// Another case to handle is if the mainfile already had more streams and the file version being restored has +// less streams, then the extra streams need to be removed from the main file. The stream names are present +// as the value in the generic attribute TypeHasAds. +func createOrOpenFile(path string, createSize int64, fileInfo *fileInfo, allowRecursiveDelete bool) (*os.File, error) { + var mainPath string + mainPath, f, err := openFileImpl(path, createSize, fileInfo) + if err != nil && fs.IsAccessDenied(err) { + // If file is readonly, clear the readonly flag by resetting the + // permissions of the file and try again + // as the metadata will be set again in the second pass and the + // readonly flag will be applied again if needed. + if err = fs.ResetPermissions(mainPath); err != nil { + return nil, err + } + if f, err = fs.OpenFile(path, fs.O_WRONLY|fs.O_NOFOLLOW, 0600); err != nil { + return nil, err + } + } else if err != nil && (errors.Is(err, syscall.ELOOP) || errors.Is(err, syscall.EISDIR)) { + // symlink or directory, try to remove it later on + f = nil + } else if err != nil { + return nil, err + } + return postCreateFile(f, path, createSize, allowRecursiveDelete, fileInfo.sparse) +} + +// openFileImpl is the actual open file implementation. +func openFileImpl(path string, createSize int64, fileInfo *fileInfo) (mainPath string, file *os.File, err error) { + if createSize >= 0 { + // File needs to be created or replaced + + //Define all the flags + var isAlreadyExists bool + var isAdsRelated, hasAds, isAds = getAdsAttributes(fileInfo.attrs) + + // This means that this is an ads related file. It either has ads streams or is an ads streams + + var mainPath string + if isAds { + mainPath = restic.TrimAds(path) + } else { + mainPath = path + } + if isAdsRelated { + // Get or create a mutex based on the main file path + mutex := GetOrCreateMutex(mainPath) + mutex.Lock() + defer mutex.Unlock() + // Making sure the code below doesn't execute concurrently for the main file and any of the ads files + } + + if err != nil { + return mainPath, nil, err + } + // First check if file already exists + file, err = openFile(path) + if err == nil { + // File already exists + isAlreadyExists = true + } else if !os.IsNotExist(err) { + // Any error other that IsNotExist error, then do not continue. + // If the error was because access is denied, + // the calling method will try to check if the file is readonly and if so, it tries to + // remove the readonly attribute and call this openFileImpl method again once. + // If this method throws access denied again, then it stops trying and return the error. + return mainPath, nil, err + } + //At this point readonly flag is already handled and we need not consider it anymore. + file, err = handleCreateFile(path, file, isAdsRelated, hasAds, isAds, isAlreadyExists) + } else { + // File is already created. For subsequent writes, only use os.O_WRONLY flag. + file, err = openFile(path) + } + + return mainPath, file, err +} + +// handleCreateFile handles all the various combination of states while creating the file if needed. +func handleCreateFile(path string, fileIn *os.File, isAdsRelated, hasAds, isAds, isAlreadyExists bool) (file *os.File, err error) { + if !isAdsRelated { + // This is the simplest case where ADS files are not involved. + file, err = handleCreateFileNonAds(path, fileIn, isAlreadyExists) + } else { + // This is a complex case needing coordination between the main file and the ads files. + file, err = handleCreateFileAds(path, fileIn, hasAds, isAds, isAlreadyExists) + } + + return file, err +} + +// handleCreateFileNonAds handles all the various combination of states while creating the non-ads file if needed. +func handleCreateFileNonAds(path string, fileIn *os.File, isAlreadyExists bool) (file *os.File, err error) { + // This is the simple case. + if isAlreadyExists { + // If the non-ads file already exists, return the file + // that we already created without create option. + return fileIn, nil + } else { + // If the non-ads file did not exist, try creating the file with create flag. + return os.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_NOFOLLOW, 0600) + } +} + +// handleCreateFileAds handles all the various combination of states while creating the ads related file if needed. +func handleCreateFileAds(path string, fileIn *os.File, hasAds, isAds, isAlreadyExists bool) (file *os.File, err error) { + // This is the simple case. We do not need to change the encryption attribute. + if isAlreadyExists { + // If the ads related file already exists and no change to encryption, return the file + // that we already created without create option. + return fileIn, nil + } else { + // If the ads related file did not exist, first check if it is a hasAds or isAds + if isAds { + // If it is an ads file, then we can simple open it with create options without worrying about overwriting. + return os.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_NOFOLLOW, 0600) + } + if hasAds { + // If it is the main file which has ads files attached, we will check again if the main file wasn't created + // since we synced. + file, err = openFile(path) + if err != nil { + if os.IsNotExist(err) { + // We confirmed that the main file still doesn't exist after syncing. + // Hence creating the file with the create flag. + // Directly open the main file with create option as it should not be encrypted. + return os.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_NOFOLLOW, 0600) + } else { + // Some other error occured so stop processing and return it. + return nil, err + } + } else { + // This means that the main file exists now and requires no change to encryption. Simply return it. + return file, err + } + } + return nil, errors.New("invalid case for ads same file encryption") + } +} + +// Helper methods + +var pathMutexMap = PathMutexMap{ + mutex: make(map[string]*sync.Mutex), +} + +// PathMutexMap represents a map of mutexes, where each path maps to a unique mutex. +type PathMutexMap struct { + mu sync.RWMutex + mutex map[string]*sync.Mutex +} + +// CleanupPath performs clean up for the specified path. +func CleanupPath(path string) { + removeMutex(path) +} + +// removeMutex removes the mutex for the specified path. +func removeMutex(path string) { + path = restic.TrimAds(path) + pathMutexMap.mu.Lock() + defer pathMutexMap.mu.Unlock() + + // Delete the mutex from the map + delete(pathMutexMap.mutex, path) +} + +// Cleanup performs cleanup for all paths. +// It clears all the mutexes in the map. +func Cleanup() { + pathMutexMap.mu.Lock() + defer pathMutexMap.mu.Unlock() + // Iterate over the map and remove each mutex + for path, mutex := range pathMutexMap.mutex { + // You can optionally do additional cleanup or release resources associated with the mutex + mutex.Lock() + // Delete the mutex from the map + delete(pathMutexMap.mutex, path) + mutex.Unlock() + } +} + +// GetOrCreateMutex returns the mutex associated with the given path. +// If the mutex doesn't exist, it creates a new one. +func GetOrCreateMutex(path string) *sync.Mutex { + pathMutexMap.mu.RLock() + mutex, ok := pathMutexMap.mutex[path] + pathMutexMap.mu.RUnlock() + + if !ok { + // The mutex doesn't exist, upgrade the lock and create a new one + pathMutexMap.mu.Lock() + defer pathMutexMap.mu.Unlock() + + // Double-check if another goroutine has created the mutex + if mutex, ok = pathMutexMap.mutex[path]; !ok { + mutex = &sync.Mutex{} + pathMutexMap.mutex[path] = mutex + } + } + return mutex +} + +// getAdsAttributes gets all the ads related attributes. +func getAdsAttributes(attrs map[restic.GenericAttributeType]json.RawMessage) (isAdsRelated, hasAds, isAds bool) { + if len(attrs) > 0 { + adsBytes := attrs[restic.TypeHasADS] + hasAds = adsBytes != nil + isAds = string(attrs[restic.TypeIsADS]) != "true" + } + isAdsRelated = hasAds || isAds + return isAdsRelated, hasAds, isAds +}