Add support for backing up ADS

internal/archiver/archiver.go

@@ -242,21 +242,21 @@ func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s I
 	case restic.NodeTypeDir:
 		switch {
 		case previous == nil:
-			arch.summary.Dirs.New++
+			arch.summary.Dirs.incrementNewFiles(current)
 		case previous.Equals(*current):
-			arch.summary.Dirs.Unchanged++
+			arch.summary.Dirs.incrementUnchangedFiles(current)
 		default:
-			arch.summary.Dirs.Changed++
+			arch.summary.Dirs.incrementChangedFiles(current)
 		}
 
 	case restic.NodeTypeFile:
 		switch {
 		case previous == nil:
-			arch.summary.Files.New++
+			arch.summary.Files.incrementNewFiles(current)
 		case previous.Equals(*current):
-			arch.summary.Files.Unchanged++
+			arch.summary.Files.incrementUnchangedFiles(current)
 		default:
-			arch.summary.Files.Changed++
+			arch.summary.Files.incrementChangedFiles(current)
 		}
 	}
 }
@@ -320,17 +320,20 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
 	if err != nil {
 		return futureNode{}, err
 	}
+	pathnames := arch.preProcessPaths(dir, names)
+	sort.Strings(pathnames)
 
-	nodes := make([]futureNode, 0, len(names))
+	nodes := make([]futureNode, 0, len(pathnames))
 
-	for _, name := range names {
+	for _, pathname := range pathnames {
 		// test if context has been cancelled
 		if ctx.Err() != nil {
 			debug.Log("context has been cancelled, aborting")
 			return futureNode{}, ctx.Err()
 		}
 
-		pathname := arch.FS.Join(dir, name)
+		name := getNameFromPathname(pathname)
+		pathname := arch.processPath(dir, pathname)
 		oldNode := previous.Find(name)
 		snItem := join(snPath, name)
 		fn, excluded, err := arch.save(ctx, snItem, pathname, oldNode)
@@ -343,7 +346,7 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
 				continue
 			}
 
-			return futureNode{}, err
+			return futureNode{}, errors.Wrap(err, "error saving a target (file or directory)")
 		}
 
 		if excluded {
@@ -456,7 +459,11 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
 	if err != nil {
 		return futureNode{}, false, err
 	}
-
+	//In case of windows ADS files for checking include and excludes we use the main file which has the ADS files attached.
+	//For Unix, the main file is the same as there is no ADS. So targetMain is always the same as target.
+	//After checking the exclusion for actually processing the file, we use the full file name including ads portion if any.
+	targetMain := fs.SanitizeMainFileName(target)
+	abstargetMain := fs.SanitizeMainFileName(abstarget)
 	filterError := func(err error) (futureNode, bool, error) {
 		err = arch.error(abstarget, err)
 		if err != nil {
@@ -493,16 +500,21 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
 	}()
 
 	// get file info and run remaining select functions that require file information
-	fi, err := meta.Stat()
+	fiMain, err := meta.Stat()
 	if err != nil {
 		debug.Log("lstat() for %v returned error: %v", target, err)
 		// ignore if file disappeared since it was returned by readdir
 		return filterError(filterNotExist(err))
 	}
-	if !arch.Select(abstarget, fi, arch.FS) {
+	if !arch.Select(abstargetMain, fiMain, arch.FS) {
 		debug.Log("%v is excluded", target)
 		return futureNode{}, true, nil
 	}
+	var fi *fs.ExtendedFileInfo
+	fi, shouldReturn, fn, excluded, err := arch.processTargets(target, targetMain, abstarget, *fiMain)
+	if shouldReturn {
+		return fn, excluded, err
+	}
 
 	switch {
 	case fi.Mode.IsRegular():
@@ -694,11 +706,6 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree,
 				}
 				return futureNode{}, 0, err
 			}
-
-			if err != nil {
-				return futureNode{}, 0, err
-			}
-
 			if !excluded {
 				nodes = append(nodes, fn)
 			}
@@ -762,13 +769,9 @@ func (arch *Archiver) dirPathToNode(snPath, target string) (node *restic.Node, e
 func resolveRelativeTargets(filesys fs.FS, targets []string) ([]string, error) {
 	debug.Log("targets before resolving: %v", targets)
 	result := make([]string, 0, len(targets))
+	preProcessTargets(filesys, &targets)
 	for _, target := range targets {
-		if target != "" && filesys.VolumeName(target) == target {
-			// special case to allow users to also specify a volume name "C:" instead of a path "C:\"
-			target = target + filesys.Separator()
-		} else {
-			target = filesys.Clean(target)
-		}
+		target = processTarget(filesys, target)
 		pc, _ := pathComponents(filesys, target, false)
 		if len(pc) > 0 {
 			result = append(result, target)

internal/filter/filter.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/fs"
 )
 
 // ErrBadString is returned when Match is called with the empty string as the
@@ -200,9 +201,9 @@ func match(pattern Pattern, strs []string) (matched bool, err error) {
 			for i := len(pattern.parts) - 1; i >= 0; i-- {
 				var ok bool
 				if pattern.parts[i].isSimple {
-					ok = pattern.parts[i].pattern == strs[offset+i]
+					ok = pattern.parts[i].pattern == fs.SanitizeMainFileName(strs[offset+i])
 				} else {
-					ok, err = filepath.Match(pattern.parts[i].pattern, strs[offset+i])
+					ok, err = filepath.Match(pattern.parts[i].pattern, fs.SanitizeMainFileName(strs[offset+i]))
 					if err != nil {
 						return false, errors.Wrap(err, "Match")
 					}

internal/fs/file_unix.go

@@ -48,3 +48,9 @@ func chmod(name string, mode os.FileMode) error {
 
 	return err
 }
+
+// SanitizeMainFileName will only keep the main file and remove the secondary file.
+func SanitizeMainFileName(str string) string {
+	// no-op - In case of non-windows there is no secondary file
+	return str
+}

internal/fs/file_windows.go

@@ -134,3 +134,11 @@ func openHandleForEA(nodeType restic.NodeType, path string, writeAccess bool) (h
 	}
 	return handle, err
 }
+
+// SanitizeMainFileName will only keep the main file and remove the secondary file like ADS from the name.
+func SanitizeMainFileName(str string) string {
+	// The ADS is essentially a part of the main file. So for any functionality that
+	// needs to consider the main file, like filtering, we need to derive the main file name
+	// from the ADS name.
+	return restic.TrimAds(str)
+}

internal/restorer/fileswriter.go

@@ -4,11 +4,9 @@ import (
 	"fmt"
 	"os"
 	"sync"
-	"syscall"
 
 	"github.com/cespare/xxhash/v2"
 	"github.com/restic/restic/internal/debug"
-	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
 )
 
@@ -61,27 +59,10 @@ func openFile(path string) (*os.File, error) {
 	return f, nil
 }
 
-func createFile(path string, createSize int64, sparse bool, allowRecursiveDelete bool) (*os.File, error) {
-	f, err := fs.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_NOFOLLOW, 0600)
-	if err != nil && fs.IsAccessDenied(err) {
-		// If file is readonly, clear the readonly flag by resetting the
-		// permissions of the file and try again
-		// as the metadata will be set again in the second pass and the
-		// readonly flag will be applied again if needed.
-		if err = fs.ResetPermissions(path); err != nil {
-			return nil, err
-		}
-		if f, err = fs.OpenFile(path, fs.O_WRONLY|fs.O_NOFOLLOW, 0600); err != nil {
-			return nil, err
-		}
-	} else if err != nil && (errors.Is(err, syscall.ELOOP) || errors.Is(err, syscall.EISDIR)) {
-		// symlink or directory, try to remove it later on
-		f = nil
-	} else if err != nil {
-		return nil, err
-	}
+func postCreateFile(f *os.File, path string, createSize int64, allowRecursiveDelete, sparse bool) (*os.File, error) {
 
 	var fi os.FileInfo
+	var err error
 	if f != nil {
 		// stat to check that we've opened a regular file
 		fi, err = f.Stat()
@@ -162,7 +143,7 @@ func ensureSize(f *os.File, fi os.FileInfo, createSize int64, sparse bool) (*os.
 	return f, nil
 }
 
-func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, createSize int64, sparse bool) error {
+func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, createSize int64, fileInfo *fileInfo) error {
 	bucket := &w.buckets[uint(xxhash.Sum64String(path))%uint(len(w.buckets))]
 
 	acquireWriter := func() (*partialFile, error) {
@@ -173,18 +154,12 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create
 			bucket.files[path].users++
 			return wr, nil
 		}
-		var f *os.File
-		var err error
-		if createSize >= 0 {
-			f, err = createFile(path, createSize, sparse, w.allowRecursiveDelete)
-			if err != nil {
-				return nil, err
-			}
-		} else if f, err = openFile(path); err != nil {
+		f, err := createOrOpenFile(path, createSize, fileInfo, w.allowRecursiveDelete)
+		if err != nil {
 			return nil, err
 		}
 
-		wr := &partialFile{File: f, users: 1, sparse: sparse}
+		wr := &partialFile{File: f, users: 1, sparse: fileInfo.sparse}
 		bucket.files[path] = wr
 
 		return wr, nil
@@ -196,6 +171,8 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create
 
 		if bucket.files[path].users == 1 {
 			delete(bucket.files, path)
+			//Clean up for the path
+			CleanupPath(path)
 			return wr.Close()
 		}
 		bucket.files[path].users--

internal/restorer/fileswriter_unix.go

@@ -0,0 +1,89 @@
+//go:build !windows
+// +build !windows
+
+package restorer
+
+import (
+	"os"
+	"syscall"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/fs"
+)
+
+// OpenFile opens the file with create, truncate and write only options if
+// createSize is specified greater than 0 i.e. if the file hasn't already
+// been created. Otherwise it opens the file with only write only option.
+func (fw *filesWriter) OpenFile(createSize int64, path string, fileInfo *fileInfo) (file *os.File, err error) {
+	return fw.openFile(createSize, path, fileInfo)
+}
+
+// OpenFile opens the file with create, truncate and write only options if
+// createSize is specified greater than 0 i.e. if the file hasn't already
+// been created. Otherwise it opens the file with only write only option.
+func (fw *filesWriter) openFile(createSize int64, path string, _ *fileInfo) (file *os.File, err error) {
+	if createSize >= 0 {
+		file, err = openFileWithCreate(path)
+		if fs.IsAccessDenied(err) {
+			// If file is readonly, clear the readonly flag by resetting the
+			// permissions of the file and try again
+			// as the metadata will be set again in the second pass and the
+			// readonly flag will be applied again if needed.
+			err = fs.ResetPermissions(path)
+			if err != nil {
+				return nil, err
+			}
+			file, err = openFileWithTruncWrite(path)
+		}
+	} else {
+		flags := os.O_WRONLY
+		file, err = os.OpenFile(path, flags, 0600)
+	}
+	return file, err
+}
+
+// openFileWithCreate opens the file with os.O_CREATE flag along with os.O_TRUNC and os.O_WRONLY.
+func openFileWithCreate(path string) (file *os.File, err error) {
+	flags := os.O_CREATE | os.O_TRUNC | os.O_WRONLY
+	return os.OpenFile(path, flags, 0600)
+}
+
+// openFileWithTruncWrite opens the file without os.O_CREATE flag along with os.O_TRUNC and os.O_WRONLY.
+func openFileWithTruncWrite(path string) (file *os.File, err error) {
+	flags := os.O_TRUNC | os.O_WRONLY
+	return os.OpenFile(path, flags, 0600)
+}
+
+// CleanupPath performs clean up for the specified path.
+func CleanupPath(_ string) {
+	// no-op
+}
+
+func createOrOpenFile(path string, createSize int64, fileInfo *fileInfo, allowRecursiveDelete bool) (*os.File, error) {
+	if createSize >= 0 {
+		return createFile(path, createSize, fileInfo, allowRecursiveDelete)
+	}
+	return openFile(path)
+}
+
+func createFile(path string, createSize int64, fileInfo *fileInfo, allowRecursiveDelete bool) (*os.File, error) {
+	f, err := fs.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_NOFOLLOW, 0600)
+	if err != nil && fs.IsAccessDenied(err) {
+		// If file is readonly, clear the readonly flag by resetting the
+		// permissions of the file and try again
+		// as the metadata will be set again in the second pass and the
+		// readonly flag will be applied again if needed.
+		if err = fs.ResetPermissions(path); err != nil {
+			return nil, err
+		}
+		if f, err = fs.OpenFile(path, fs.O_WRONLY|fs.O_NOFOLLOW, 0600); err != nil {
+			return nil, err
+		}
+	} else if err != nil && (errors.Is(err, syscall.ELOOP) || errors.Is(err, syscall.EISDIR)) {
+		// symlink or directory, try to remove it later on
+		f = nil
+	} else if err != nil {
+		return nil, err
+	}
+	return postCreateFile(f, path, createSize, allowRecursiveDelete, fileInfo.sparse)
+}

internal/restorer/fileswriter_windows.go

@@ -0,0 +1,237 @@
+package restorer
+
+import (
+	"encoding/json"
+	"errors"
+	"os"
+	"sync"
+	"syscall"
+
+	"github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/restic"
+)
+
+// createOrOpenFile opens the file and handles the readonly attribute and ads related logic during file creation.
+// Readonly files - if an existing file is detected as readonly we clear the flag because otherwise we cannot
+// make changes to the file. The readonly attribute would be set again in the second pass when the attributes
+// are set if the file version being restored has the readonly bit.
+// ADS files need special handling - Each stream is treated as a separate file in restic. This method is called
+// for the main file which has the streams and for each stream.
+// If the ads stream calls this method first and the main file doesn't already exist, then creating the file
+// for the streams causes the main file to automatically get created with 0 size. Hence we need to be careful
+// while creating the main file. If we blindly create it with the os.O_CREATE option, it could overwrite the
+// stream. However creating the stream with os.O_CREATE option does not overwrite the mainfile if it already
+// exists. It will simply attach the new stream to the main file if the main file existed, otherwise it will
+// create the 0 size main file.
+// Another case to handle is if the mainfile already had more streams and the file version being restored has
+// less streams, then the extra streams need to be removed from the main file. The stream names are present
+// as the value in the generic attribute TypeHasAds.
+func createOrOpenFile(path string, createSize int64, fileInfo *fileInfo, allowRecursiveDelete bool) (*os.File, error) {
+	var mainPath string
+	mainPath, f, err := openFileImpl(path, createSize, fileInfo)
+	if err != nil && fs.IsAccessDenied(err) {
+		// If file is readonly, clear the readonly flag by resetting the
+		// permissions of the file and try again
+		// as the metadata will be set again in the second pass and the
+		// readonly flag will be applied again if needed.
+		if err = fs.ResetPermissions(mainPath); err != nil {
+			return nil, err
+		}
+		if f, err = fs.OpenFile(path, fs.O_WRONLY|fs.O_NOFOLLOW, 0600); err != nil {
+			return nil, err
+		}
+	} else if err != nil && (errors.Is(err, syscall.ELOOP) || errors.Is(err, syscall.EISDIR)) {
+		// symlink or directory, try to remove it later on
+		f = nil
+	} else if err != nil {
+		return nil, err
+	}
+	return postCreateFile(f, path, createSize, allowRecursiveDelete, fileInfo.sparse)
+}
+
+// openFileImpl is the actual open file implementation.
+func openFileImpl(path string, createSize int64, fileInfo *fileInfo) (mainPath string, file *os.File, err error) {
+	if createSize >= 0 {
+		// File needs to be created or replaced
+
+		//Define all the flags
+		var isAlreadyExists bool
+		var isAdsRelated, hasAds, isAds = getAdsAttributes(fileInfo.attrs)
+
+		// This means that this is an ads related file. It either has ads streams or is an ads streams
+
+		var mainPath string
+		if isAds {
+			mainPath = restic.TrimAds(path)
+		} else {
+			mainPath = path
+		}
+		if isAdsRelated {
+			// Get or create a mutex based on the main file path
+			mutex := GetOrCreateMutex(mainPath)
+			mutex.Lock()
+			defer mutex.Unlock()
+			// Making sure the code below doesn't execute concurrently for the main file and any of the ads files
+		}
+
+		if err != nil {
+			return mainPath, nil, err
+		}
+		// First check if file already exists
+		file, err = openFile(path)
+		if err == nil {
+			// File already exists
+			isAlreadyExists = true
+		} else if !os.IsNotExist(err) {
+			// Any error other that IsNotExist error, then do not continue.
+			// If the error was because access is denied,
+			// the calling method will try to check if the file is readonly and if so, it tries to
+			// remove the readonly attribute and call this openFileImpl method again once.
+			// If this method throws access denied again, then it stops trying and return the error.
+			return mainPath, nil, err
+		}
+		//At this point readonly flag is already handled and we need not consider it anymore.
+		file, err = handleCreateFile(path, file, isAdsRelated, hasAds, isAds, isAlreadyExists)
+	} else {
+		// File is already created. For subsequent writes, only use os.O_WRONLY flag.
+		file, err = openFile(path)
+	}
+
+	return mainPath, file, err
+}
+
+// handleCreateFile handles all the various combination of states while creating the file if needed.
+func handleCreateFile(path string, fileIn *os.File, isAdsRelated, hasAds, isAds, isAlreadyExists bool) (file *os.File, err error) {
+	if !isAdsRelated {
+		// This is the simplest case where ADS files are not involved.
+		file, err = handleCreateFileNonAds(path, fileIn, isAlreadyExists)
+	} else {
+		// This is a complex case needing coordination between the main file and the ads files.
+		file, err = handleCreateFileAds(path, fileIn, hasAds, isAds, isAlreadyExists)
+	}
+
+	return file, err
+}
+
+// handleCreateFileNonAds handles all the various combination of states while creating the non-ads file if needed.
+func handleCreateFileNonAds(path string, fileIn *os.File, isAlreadyExists bool) (file *os.File, err error) {
+	// This is the simple case.
+	if isAlreadyExists {
+		// If the non-ads file already exists, return the file
+		// that we already created without create option.
+		return fileIn, nil
+	} else {
+		// If the non-ads file did not exist, try creating the file with create flag.
+		return os.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_NOFOLLOW, 0600)
+	}
+}
+
+// handleCreateFileAds handles all the various combination of states while creating the ads related file if needed.
+func handleCreateFileAds(path string, fileIn *os.File, hasAds, isAds, isAlreadyExists bool) (file *os.File, err error) {
+	// This is the simple case. We do not need to change the encryption attribute.
+	if isAlreadyExists {
+		// If the ads related file already exists and no change to encryption, return the file
+		// that we already created without create option.
+		return fileIn, nil
+	} else {
+		// If the ads related file did not exist, first check if it is a hasAds or isAds
+		if isAds {
+			// If it is an ads file, then we can simple open it with create options without worrying about overwriting.
+			return os.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_NOFOLLOW, 0600)
+		}
+		if hasAds {
+			// If it is the main file which has ads files attached, we will check again if the main file wasn't created
+			// since we synced.
+			file, err = openFile(path)
+			if err != nil {
+				if os.IsNotExist(err) {
+					// We confirmed that the main file still doesn't exist after syncing.
+					// Hence creating the file with the create flag.
+					// Directly open the main file with create option as it should not be encrypted.
+					return os.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_NOFOLLOW, 0600)
+				} else {
+					// Some other error occured so stop processing and return it.
+					return nil, err
+				}
+			} else {
+				// This means that the main file exists now and requires no change to encryption. Simply return it.
+				return file, err
+			}
+		}
+		return nil, errors.New("invalid case for ads same file encryption")
+	}
+}
+
+// Helper methods
+
+var pathMutexMap = PathMutexMap{
+	mutex: make(map[string]*sync.Mutex),
+}
+
+// PathMutexMap represents a map of mutexes, where each path maps to a unique mutex.
+type PathMutexMap struct {
+	mu    sync.RWMutex
+	mutex map[string]*sync.Mutex
+}
+
+// CleanupPath performs clean up for the specified path.
+func CleanupPath(path string) {
+	removeMutex(path)
+}
+
+// removeMutex removes the mutex for the specified path.
+func removeMutex(path string) {
+	path = restic.TrimAds(path)
+	pathMutexMap.mu.Lock()
+	defer pathMutexMap.mu.Unlock()
+
+	// Delete the mutex from the map
+	delete(pathMutexMap.mutex, path)
+}
+
+// Cleanup performs cleanup for all paths.
+// It clears all the mutexes in the map.
+func Cleanup() {
+	pathMutexMap.mu.Lock()
+	defer pathMutexMap.mu.Unlock()
+	// Iterate over the map and remove each mutex
+	for path, mutex := range pathMutexMap.mutex {
+		// You can optionally do additional cleanup or release resources associated with the mutex
+		mutex.Lock()
+		// Delete the mutex from the map
+		delete(pathMutexMap.mutex, path)
+		mutex.Unlock()
+	}
+}
+
+// GetOrCreateMutex returns the mutex associated with the given path.
+// If the mutex doesn't exist, it creates a new one.
+func GetOrCreateMutex(path string) *sync.Mutex {
+	pathMutexMap.mu.RLock()
+	mutex, ok := pathMutexMap.mutex[path]
+	pathMutexMap.mu.RUnlock()
+
+	if !ok {
+		// The mutex doesn't exist, upgrade the lock and create a new one
+		pathMutexMap.mu.Lock()
+		defer pathMutexMap.mu.Unlock()
+
+		// Double-check if another goroutine has created the mutex
+		if mutex, ok = pathMutexMap.mutex[path]; !ok {
+			mutex = &sync.Mutex{}
+			pathMutexMap.mutex[path] = mutex
+		}
+	}
+	return mutex
+}
+
+// getAdsAttributes gets all the ads related attributes.
+func getAdsAttributes(attrs map[restic.GenericAttributeType]json.RawMessage) (isAdsRelated, hasAds, isAds bool) {
+	if len(attrs) > 0 {
+		adsBytes := attrs[restic.TypeHasADS]
+		hasAds = adsBytes != nil
+		isAds = string(attrs[restic.TypeIsADS]) != "true"
+	}
+	isAdsRelated = hasAds || isAds
+	return isAdsRelated, hasAds, isAds
+}