1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2025-03-09 00:00:02 +01:00

Add support for backing up ADS

This commit is contained in:
aneesh-n 2024-12-03 17:57:30 +05:30
parent 0cff89de41
commit 691890c502
No known key found for this signature in database
GPG key ID: 6F5A52831C046F44
7 changed files with 378 additions and 57 deletions

View file

@ -242,21 +242,21 @@ func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s I
case restic.NodeTypeDir:
switch {
case previous == nil:
arch.summary.Dirs.New++
arch.summary.Dirs.incrementNewFiles(current)
case previous.Equals(*current):
arch.summary.Dirs.Unchanged++
arch.summary.Dirs.incrementUnchangedFiles(current)
default:
arch.summary.Dirs.Changed++
arch.summary.Dirs.incrementChangedFiles(current)
}
case restic.NodeTypeFile:
switch {
case previous == nil:
arch.summary.Files.New++
arch.summary.Files.incrementNewFiles(current)
case previous.Equals(*current):
arch.summary.Files.Unchanged++
arch.summary.Files.incrementUnchangedFiles(current)
default:
arch.summary.Files.Changed++
arch.summary.Files.incrementChangedFiles(current)
}
}
}
@ -320,17 +320,20 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
if err != nil {
return futureNode{}, err
}
pathnames := arch.preProcessPaths(dir, names)
sort.Strings(pathnames)
nodes := make([]futureNode, 0, len(names))
nodes := make([]futureNode, 0, len(pathnames))
for _, name := range names {
for _, pathname := range pathnames {
// test if context has been cancelled
if ctx.Err() != nil {
debug.Log("context has been cancelled, aborting")
return futureNode{}, ctx.Err()
}
pathname := arch.FS.Join(dir, name)
name := getNameFromPathname(pathname)
pathname := arch.processPath(dir, pathname)
oldNode := previous.Find(name)
snItem := join(snPath, name)
fn, excluded, err := arch.save(ctx, snItem, pathname, oldNode)
@ -343,7 +346,7 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
continue
}
return futureNode{}, err
return futureNode{}, errors.Wrap(err, "error saving a target (file or directory)")
}
if excluded {
@ -456,7 +459,11 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
if err != nil {
return futureNode{}, false, err
}
//In case of windows ADS files for checking include and excludes we use the main file which has the ADS files attached.
//For Unix, the main file is the same as there is no ADS. So targetMain is always the same as target.
//After checking the exclusion for actually processing the file, we use the full file name including ads portion if any.
targetMain := fs.SanitizeMainFileName(target)
abstargetMain := fs.SanitizeMainFileName(abstarget)
filterError := func(err error) (futureNode, bool, error) {
err = arch.error(abstarget, err)
if err != nil {
@ -493,16 +500,21 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
}()
// get file info and run remaining select functions that require file information
fi, err := meta.Stat()
fiMain, err := meta.Stat()
if err != nil {
debug.Log("lstat() for %v returned error: %v", target, err)
// ignore if file disappeared since it was returned by readdir
return filterError(filterNotExist(err))
}
if !arch.Select(abstarget, fi, arch.FS) {
if !arch.Select(abstargetMain, fiMain, arch.FS) {
debug.Log("%v is excluded", target)
return futureNode{}, true, nil
}
var fi *fs.ExtendedFileInfo
fi, shouldReturn, fn, excluded, err := arch.processTargets(target, targetMain, abstarget, *fiMain)
if shouldReturn {
return fn, excluded, err
}
switch {
case fi.Mode.IsRegular():
@ -694,11 +706,6 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree,
}
return futureNode{}, 0, err
}
if err != nil {
return futureNode{}, 0, err
}
if !excluded {
nodes = append(nodes, fn)
}
@ -762,13 +769,9 @@ func (arch *Archiver) dirPathToNode(snPath, target string) (node *restic.Node, e
func resolveRelativeTargets(filesys fs.FS, targets []string) ([]string, error) {
debug.Log("targets before resolving: %v", targets)
result := make([]string, 0, len(targets))
preProcessTargets(filesys, &targets)
for _, target := range targets {
if target != "" && filesys.VolumeName(target) == target {
// special case to allow users to also specify a volume name "C:" instead of a path "C:\"
target = target + filesys.Separator()
} else {
target = filesys.Clean(target)
}
target = processTarget(filesys, target)
pc, _ := pathComponents(filesys, target, false)
if len(pc) > 0 {
result = append(result, target)

View file

@ -5,6 +5,7 @@ import (
"strings"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
)
// ErrBadString is returned when Match is called with the empty string as the
@ -200,9 +201,9 @@ func match(pattern Pattern, strs []string) (matched bool, err error) {
for i := len(pattern.parts) - 1; i >= 0; i-- {
var ok bool
if pattern.parts[i].isSimple {
ok = pattern.parts[i].pattern == strs[offset+i]
ok = pattern.parts[i].pattern == fs.SanitizeMainFileName(strs[offset+i])
} else {
ok, err = filepath.Match(pattern.parts[i].pattern, strs[offset+i])
ok, err = filepath.Match(pattern.parts[i].pattern, fs.SanitizeMainFileName(strs[offset+i]))
if err != nil {
return false, errors.Wrap(err, "Match")
}

View file

@ -48,3 +48,9 @@ func chmod(name string, mode os.FileMode) error {
return err
}
// SanitizeMainFileName will only keep the main file and remove the secondary file.
func SanitizeMainFileName(str string) string {
// no-op - In case of non-windows there is no secondary file
return str
}

View file

@ -134,3 +134,11 @@ func openHandleForEA(nodeType restic.NodeType, path string, writeAccess bool) (h
}
return handle, err
}
// SanitizeMainFileName will only keep the main file and remove the secondary file like ADS from the name.
func SanitizeMainFileName(str string) string {
// The ADS is essentially a part of the main file. So for any functionality that
// needs to consider the main file, like filtering, we need to derive the main file name
// from the ADS name.
return restic.TrimAds(str)
}

View file

@ -4,11 +4,9 @@ import (
"fmt"
"os"
"sync"
"syscall"
"github.com/cespare/xxhash/v2"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
)
@ -61,27 +59,10 @@ func openFile(path string) (*os.File, error) {
return f, nil
}
func createFile(path string, createSize int64, sparse bool, allowRecursiveDelete bool) (*os.File, error) {
f, err := fs.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_NOFOLLOW, 0600)
if err != nil && fs.IsAccessDenied(err) {
// If file is readonly, clear the readonly flag by resetting the
// permissions of the file and try again
// as the metadata will be set again in the second pass and the
// readonly flag will be applied again if needed.
if err = fs.ResetPermissions(path); err != nil {
return nil, err
}
if f, err = fs.OpenFile(path, fs.O_WRONLY|fs.O_NOFOLLOW, 0600); err != nil {
return nil, err
}
} else if err != nil && (errors.Is(err, syscall.ELOOP) || errors.Is(err, syscall.EISDIR)) {
// symlink or directory, try to remove it later on
f = nil
} else if err != nil {
return nil, err
}
func postCreateFile(f *os.File, path string, createSize int64, allowRecursiveDelete, sparse bool) (*os.File, error) {
var fi os.FileInfo
var err error
if f != nil {
// stat to check that we've opened a regular file
fi, err = f.Stat()
@ -162,7 +143,7 @@ func ensureSize(f *os.File, fi os.FileInfo, createSize int64, sparse bool) (*os.
return f, nil
}
func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, createSize int64, sparse bool) error {
func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, createSize int64, fileInfo *fileInfo) error {
bucket := &w.buckets[uint(xxhash.Sum64String(path))%uint(len(w.buckets))]
acquireWriter := func() (*partialFile, error) {
@ -173,18 +154,12 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create
bucket.files[path].users++
return wr, nil
}
var f *os.File
var err error
if createSize >= 0 {
f, err = createFile(path, createSize, sparse, w.allowRecursiveDelete)
if err != nil {
return nil, err
}
} else if f, err = openFile(path); err != nil {
f, err := createOrOpenFile(path, createSize, fileInfo, w.allowRecursiveDelete)
if err != nil {
return nil, err
}
wr := &partialFile{File: f, users: 1, sparse: sparse}
wr := &partialFile{File: f, users: 1, sparse: fileInfo.sparse}
bucket.files[path] = wr
return wr, nil
@ -196,6 +171,8 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create
if bucket.files[path].users == 1 {
delete(bucket.files, path)
//Clean up for the path
CleanupPath(path)
return wr.Close()
}
bucket.files[path].users--

View file

@ -0,0 +1,89 @@
//go:build !windows
// +build !windows
package restorer
import (
"os"
"syscall"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
)
// OpenFile opens the file with create, truncate and write only options if
// createSize is specified greater than 0 i.e. if the file hasn't already
// been created. Otherwise it opens the file with only write only option.
func (fw *filesWriter) OpenFile(createSize int64, path string, fileInfo *fileInfo) (file *os.File, err error) {
return fw.openFile(createSize, path, fileInfo)
}
// OpenFile opens the file with create, truncate and write only options if
// createSize is specified greater than 0 i.e. if the file hasn't already
// been created. Otherwise it opens the file with only write only option.
func (fw *filesWriter) openFile(createSize int64, path string, _ *fileInfo) (file *os.File, err error) {
if createSize >= 0 {
file, err = openFileWithCreate(path)
if fs.IsAccessDenied(err) {
// If file is readonly, clear the readonly flag by resetting the
// permissions of the file and try again
// as the metadata will be set again in the second pass and the
// readonly flag will be applied again if needed.
err = fs.ResetPermissions(path)
if err != nil {
return nil, err
}
file, err = openFileWithTruncWrite(path)
}
} else {
flags := os.O_WRONLY
file, err = os.OpenFile(path, flags, 0600)
}
return file, err
}
// openFileWithCreate opens the file with os.O_CREATE flag along with os.O_TRUNC and os.O_WRONLY.
func openFileWithCreate(path string) (file *os.File, err error) {
flags := os.O_CREATE | os.O_TRUNC | os.O_WRONLY
return os.OpenFile(path, flags, 0600)
}
// openFileWithTruncWrite opens the file without os.O_CREATE flag along with os.O_TRUNC and os.O_WRONLY.
func openFileWithTruncWrite(path string) (file *os.File, err error) {
flags := os.O_TRUNC | os.O_WRONLY
return os.OpenFile(path, flags, 0600)
}
// CleanupPath performs clean up for the specified path.
func CleanupPath(_ string) {
// no-op
}
func createOrOpenFile(path string, createSize int64, fileInfo *fileInfo, allowRecursiveDelete bool) (*os.File, error) {
if createSize >= 0 {
return createFile(path, createSize, fileInfo, allowRecursiveDelete)
}
return openFile(path)
}
func createFile(path string, createSize int64, fileInfo *fileInfo, allowRecursiveDelete bool) (*os.File, error) {
f, err := fs.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_NOFOLLOW, 0600)
if err != nil && fs.IsAccessDenied(err) {
// If file is readonly, clear the readonly flag by resetting the
// permissions of the file and try again
// as the metadata will be set again in the second pass and the
// readonly flag will be applied again if needed.
if err = fs.ResetPermissions(path); err != nil {
return nil, err
}
if f, err = fs.OpenFile(path, fs.O_WRONLY|fs.O_NOFOLLOW, 0600); err != nil {
return nil, err
}
} else if err != nil && (errors.Is(err, syscall.ELOOP) || errors.Is(err, syscall.EISDIR)) {
// symlink or directory, try to remove it later on
f = nil
} else if err != nil {
return nil, err
}
return postCreateFile(f, path, createSize, allowRecursiveDelete, fileInfo.sparse)
}

View file

@ -0,0 +1,237 @@
package restorer
import (
"encoding/json"
"errors"
"os"
"sync"
"syscall"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
)
// createOrOpenFile opens the file and handles the readonly attribute and ads related logic during file creation.
// Readonly files - if an existing file is detected as readonly we clear the flag because otherwise we cannot
// make changes to the file. The readonly attribute would be set again in the second pass when the attributes
// are set if the file version being restored has the readonly bit.
// ADS files need special handling - Each stream is treated as a separate file in restic. This method is called
// for the main file which has the streams and for each stream.
// If the ads stream calls this method first and the main file doesn't already exist, then creating the file
// for the streams causes the main file to automatically get created with 0 size. Hence we need to be careful
// while creating the main file. If we blindly create it with the os.O_CREATE option, it could overwrite the
// stream. However creating the stream with os.O_CREATE option does not overwrite the mainfile if it already
// exists. It will simply attach the new stream to the main file if the main file existed, otherwise it will
// create the 0 size main file.
// Another case to handle is if the mainfile already had more streams and the file version being restored has
// less streams, then the extra streams need to be removed from the main file. The stream names are present
// as the value in the generic attribute TypeHasAds.
func createOrOpenFile(path string, createSize int64, fileInfo *fileInfo, allowRecursiveDelete bool) (*os.File, error) {
var mainPath string
mainPath, f, err := openFileImpl(path, createSize, fileInfo)
if err != nil && fs.IsAccessDenied(err) {
// If file is readonly, clear the readonly flag by resetting the
// permissions of the file and try again
// as the metadata will be set again in the second pass and the
// readonly flag will be applied again if needed.
if err = fs.ResetPermissions(mainPath); err != nil {
return nil, err
}
if f, err = fs.OpenFile(path, fs.O_WRONLY|fs.O_NOFOLLOW, 0600); err != nil {
return nil, err
}
} else if err != nil && (errors.Is(err, syscall.ELOOP) || errors.Is(err, syscall.EISDIR)) {
// symlink or directory, try to remove it later on
f = nil
} else if err != nil {
return nil, err
}
return postCreateFile(f, path, createSize, allowRecursiveDelete, fileInfo.sparse)
}
// openFileImpl is the actual open file implementation.
func openFileImpl(path string, createSize int64, fileInfo *fileInfo) (mainPath string, file *os.File, err error) {
if createSize >= 0 {
// File needs to be created or replaced
//Define all the flags
var isAlreadyExists bool
var isAdsRelated, hasAds, isAds = getAdsAttributes(fileInfo.attrs)
// This means that this is an ads related file. It either has ads streams or is an ads streams
var mainPath string
if isAds {
mainPath = restic.TrimAds(path)
} else {
mainPath = path
}
if isAdsRelated {
// Get or create a mutex based on the main file path
mutex := GetOrCreateMutex(mainPath)
mutex.Lock()
defer mutex.Unlock()
// Making sure the code below doesn't execute concurrently for the main file and any of the ads files
}
if err != nil {
return mainPath, nil, err
}
// First check if file already exists
file, err = openFile(path)
if err == nil {
// File already exists
isAlreadyExists = true
} else if !os.IsNotExist(err) {
// Any error other that IsNotExist error, then do not continue.
// If the error was because access is denied,
// the calling method will try to check if the file is readonly and if so, it tries to
// remove the readonly attribute and call this openFileImpl method again once.
// If this method throws access denied again, then it stops trying and return the error.
return mainPath, nil, err
}
//At this point readonly flag is already handled and we need not consider it anymore.
file, err = handleCreateFile(path, file, isAdsRelated, hasAds, isAds, isAlreadyExists)
} else {
// File is already created. For subsequent writes, only use os.O_WRONLY flag.
file, err = openFile(path)
}
return mainPath, file, err
}
// handleCreateFile handles all the various combination of states while creating the file if needed.
func handleCreateFile(path string, fileIn *os.File, isAdsRelated, hasAds, isAds, isAlreadyExists bool) (file *os.File, err error) {
if !isAdsRelated {
// This is the simplest case where ADS files are not involved.
file, err = handleCreateFileNonAds(path, fileIn, isAlreadyExists)
} else {
// This is a complex case needing coordination between the main file and the ads files.
file, err = handleCreateFileAds(path, fileIn, hasAds, isAds, isAlreadyExists)
}
return file, err
}
// handleCreateFileNonAds handles all the various combination of states while creating the non-ads file if needed.
func handleCreateFileNonAds(path string, fileIn *os.File, isAlreadyExists bool) (file *os.File, err error) {
// This is the simple case.
if isAlreadyExists {
// If the non-ads file already exists, return the file
// that we already created without create option.
return fileIn, nil
} else {
// If the non-ads file did not exist, try creating the file with create flag.
return os.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_NOFOLLOW, 0600)
}
}
// handleCreateFileAds handles all the various combination of states while creating the ads related file if needed.
func handleCreateFileAds(path string, fileIn *os.File, hasAds, isAds, isAlreadyExists bool) (file *os.File, err error) {
// This is the simple case. We do not need to change the encryption attribute.
if isAlreadyExists {
// If the ads related file already exists and no change to encryption, return the file
// that we already created without create option.
return fileIn, nil
} else {
// If the ads related file did not exist, first check if it is a hasAds or isAds
if isAds {
// If it is an ads file, then we can simple open it with create options without worrying about overwriting.
return os.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_NOFOLLOW, 0600)
}
if hasAds {
// If it is the main file which has ads files attached, we will check again if the main file wasn't created
// since we synced.
file, err = openFile(path)
if err != nil {
if os.IsNotExist(err) {
// We confirmed that the main file still doesn't exist after syncing.
// Hence creating the file with the create flag.
// Directly open the main file with create option as it should not be encrypted.
return os.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_NOFOLLOW, 0600)
} else {
// Some other error occured so stop processing and return it.
return nil, err
}
} else {
// This means that the main file exists now and requires no change to encryption. Simply return it.
return file, err
}
}
return nil, errors.New("invalid case for ads same file encryption")
}
}
// Helper methods
var pathMutexMap = PathMutexMap{
mutex: make(map[string]*sync.Mutex),
}
// PathMutexMap represents a map of mutexes, where each path maps to a unique mutex.
type PathMutexMap struct {
mu sync.RWMutex
mutex map[string]*sync.Mutex
}
// CleanupPath performs clean up for the specified path.
func CleanupPath(path string) {
removeMutex(path)
}
// removeMutex removes the mutex for the specified path.
func removeMutex(path string) {
path = restic.TrimAds(path)
pathMutexMap.mu.Lock()
defer pathMutexMap.mu.Unlock()
// Delete the mutex from the map
delete(pathMutexMap.mutex, path)
}
// Cleanup performs cleanup for all paths.
// It clears all the mutexes in the map.
func Cleanup() {
pathMutexMap.mu.Lock()
defer pathMutexMap.mu.Unlock()
// Iterate over the map and remove each mutex
for path, mutex := range pathMutexMap.mutex {
// You can optionally do additional cleanup or release resources associated with the mutex
mutex.Lock()
// Delete the mutex from the map
delete(pathMutexMap.mutex, path)
mutex.Unlock()
}
}
// GetOrCreateMutex returns the mutex associated with the given path.
// If the mutex doesn't exist, it creates a new one.
func GetOrCreateMutex(path string) *sync.Mutex {
pathMutexMap.mu.RLock()
mutex, ok := pathMutexMap.mutex[path]
pathMutexMap.mu.RUnlock()
if !ok {
// The mutex doesn't exist, upgrade the lock and create a new one
pathMutexMap.mu.Lock()
defer pathMutexMap.mu.Unlock()
// Double-check if another goroutine has created the mutex
if mutex, ok = pathMutexMap.mutex[path]; !ok {
mutex = &sync.Mutex{}
pathMutexMap.mutex[path] = mutex
}
}
return mutex
}
// getAdsAttributes gets all the ads related attributes.
func getAdsAttributes(attrs map[restic.GenericAttributeType]json.RawMessage) (isAdsRelated, hasAds, isAds bool) {
if len(attrs) > 0 {
adsBytes := attrs[restic.TypeHasADS]
hasAds = adsBytes != nil
isAds = string(attrs[restic.TypeIsADS]) != "true"
}
isAdsRelated = hasAds || isAds
return isAdsRelated, hasAds, isAds
}