From 07275850440b316f26a87b0c53730a79709bfc78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Phillipp=20R=C3=B6ll?= Date: Sat, 14 Sep 2024 12:13:55 +0200 Subject: [PATCH] initial commit --- cmd/restic/cmd_dump.go | 135 ++++++++++++++++++++++++++--------- internal/dump/common.go | 36 ++++++++-- internal/dump/common_test.go | 2 + internal/dump/tar.go | 20 ++---- internal/dump/zip.go | 17 ++--- 5 files changed, 147 insertions(+), 63 deletions(-) diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index 6b7f8d012..172861870 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -74,56 +74,110 @@ func splitPath(p string) []string { return append(s, f) } -func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, canWriteArchiveFunc func() error) error { +func cleanupPathList(pathComponentsList [][]string) [][]string { + // Build a set of paths for quick lookup + pathSet := make(map[string]struct{}) + + for _, splittedPath := range pathComponentsList { + pathStr := path.Join(splittedPath...) + pathSet[pathStr] = struct{}{} + } + + // Filter out subpaths that are covered by parent directories + filteredList := [][]string{} + + for _, splittedPath := range pathComponentsList { + isCovered := false + // Check if any prefix of the path exists in the pathSet + for i := len(splittedPath) - 1; i >= 1; i-- { + prefixPath := path.Join(splittedPath[0:i]...) + if _, exists := pathSet[prefixPath]; exists { + isCovered = true + break + } + } + if !isCovered { + filteredList = append(filteredList, splittedPath) + } + } + + return filteredList +} + +func filterPathComponents(pathComponentsList [][]string, prefix string) [][]string { + var filteredList [][]string + for _, pathComponents := range pathComponentsList { + if len(pathComponents) > 1 && pathComponents[0] == prefix { + filteredList = append(filteredList, pathComponents[1:]) + } + } + return filteredList +} + +func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoader, prefix string, pathComponentsList [][]string, d *dump.Dumper, canWriteArchiveFunc func() error) error { // If we print / we need to assume that there are multiple nodes at that // level in the tree. - if pathComponents[0] == "" { + if pathComponentsList[0][0] == "" { if err := canWriteArchiveFunc(); err != nil { return err } return d.DumpTree(ctx, tree, "/") } - item := filepath.Join(prefix, pathComponents[0]) - l := len(pathComponents) for _, node := range tree.Nodes { if ctx.Err() != nil { return ctx.Err() } - // If dumping something in the highest level it will just take the - // first item it finds and dump that according to the switch case below. - if node.Name == pathComponents[0] { - switch { - case l == 1 && node.Type == restic.NodeTypeFile: - return d.WriteNode(ctx, node) - case l > 1 && node.Type == restic.NodeTypeDir: - subtree, err := restic.LoadTree(ctx, repo, *node.Subtree) - if err != nil { - return errors.Wrapf(err, "cannot load subtree for %q", item) + out: + // iterate over all pathComponents + for _, pathComponents := range pathComponentsList { + item := filepath.Join(prefix, pathComponents[0]) + l := len(pathComponents) + + // If dumping something in the highest level it will just take the + // first item it finds and dump that according to the switch case below. + if node.Name == pathComponents[0] { + switch { + case l == 1 && node.Type == restic.NodeTypeFile: + d.WriteNode(ctx, node) + break out + case l > 1 && node.Type == restic.NodeTypeDir: + subtree, err := restic.LoadTree(ctx, repo, *node.Subtree) + if err != nil { + return errors.Wrapf(err, "cannot load subtree for %q", item) + } + + newComponentsList := filterPathComponents(pathComponentsList, node.Name) + + printFromTree(ctx, subtree, repo, item, newComponentsList, d, canWriteArchiveFunc) + break out + case node.Type == restic.NodeTypeDir: + if err := canWriteArchiveFunc(); err != nil { + return err + } + subtree, err := restic.LoadTree(ctx, repo, *node.Subtree) + if err != nil { + return err + } + d.DumpTree(ctx, subtree, item) + break out + case l > 1: + return fmt.Errorf("%q should be a dir, but is a %q", item, node.Type) + case node.Type != restic.NodeTypeFile: + return fmt.Errorf("%q should be a file, but is a %q", item, node.Type) } - return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d, canWriteArchiveFunc) - case node.Type == restic.NodeTypeDir: - if err := canWriteArchiveFunc(); err != nil { - return err - } - subtree, err := restic.LoadTree(ctx, repo, *node.Subtree) - if err != nil { - return err - } - return d.DumpTree(ctx, subtree, item) - case l > 1: - return fmt.Errorf("%q should be a dir, but is a %q", item, node.Type) - case node.Type != restic.NodeTypeFile: - return fmt.Errorf("%q should be a file, but is a %q", item, node.Type) } + } } - return fmt.Errorf("path %q not found in snapshot", item) + + return nil + //return fmt.Errorf("path %q not found in snapshot", item) } func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []string) error { - if len(args) != 2 { + if len(args) < 2 { return errors.Fatal("no file and no snapshot ID specified") } @@ -134,11 +188,8 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] } snapshotIDString := args[0] - pathToPrint := args[1] - debug.Log("dump file %q from %q", pathToPrint, snapshotIDString) - - splittedPath := splitPath(path.Clean(pathToPrint)) + debug.Log("dump file started for %d paths from %q", (len(args) - 1), snapshotIDString) ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock) if err != nil { @@ -188,7 +239,21 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] } d := dump.New(opts.Archive, repo, outputFileWriter) - err = printFromTree(ctx, tree, repo, "/", splittedPath, d, canWriteArchiveFunc) + defer d.Close() + + var splittedPathList [][]string + + for i := 1; i < len(args); i++ { + pathToPrint := args[i] + debug.Log("dump file %q from %q", pathToPrint, snapshotIDString) + + splittedPath := splitPath(path.Clean(pathToPrint)) + splittedPathList = append(splittedPathList, splittedPath) + } + + splittedPathList = cleanupPathList(splittedPathList) + + err = printFromTree(ctx, tree, repo, "/", splittedPathList, d, canWriteArchiveFunc) if err != nil { return errors.Fatalf("cannot dump file: %v", err) } diff --git a/internal/dump/common.go b/internal/dump/common.go index 4bc404fe0..5462c3d79 100644 --- a/internal/dump/common.go +++ b/internal/dump/common.go @@ -1,6 +1,9 @@ package dump import ( + "archive/tar" + "archive/zip" + "compress/gzip" "context" "io" "path" @@ -15,10 +18,13 @@ import ( // A Dumper writes trees and files from a repository to a Writer // in an archive format. type Dumper struct { - cache *bloblru.Cache - format string - repo restic.Loader - w io.Writer + cache *bloblru.Cache + format string + repo restic.Loader + w io.Writer + gzipWriter *gzip.Writer + zipWriter *zip.Writer + tarWriter *tar.Writer } func New(format string, repo restic.Loader, w io.Writer) *Dumper { @@ -30,6 +36,28 @@ func New(format string, repo restic.Loader, w io.Writer) *Dumper { } } +func (d *Dumper) Close() error { + if d.tarWriter != nil { + err := d.tarWriter.Close() + if err != nil { + return errors.Wrap(err, "Close tarWriter") + } + + if d.gzipWriter != nil { + err := d.gzipWriter.Close() + if err != nil { + return errors.Wrap(err, "Close gzipWriter") + } + } + } + + if d.zipWriter != nil { + return d.zipWriter.Close() + } + + return nil +} + func (d *Dumper) DumpTree(ctx context.Context, tree *restic.Tree, rootPath string) error { ctx, cancel := context.WithCancel(ctx) defer cancel() diff --git a/internal/dump/common_test.go b/internal/dump/common_test.go index afd19df63..03f6f03e2 100644 --- a/internal/dump/common_test.go +++ b/internal/dump/common_test.go @@ -89,6 +89,8 @@ func WriteTest(t *testing.T, format string, cd CheckDump) { if err := d.DumpTree(ctx, tree, tt.target); err != nil { t.Fatalf("Dumper.Run error = %v", err) } + d.Close() + if err := cd(t, tmpdir, dst); err != nil { t.Errorf("WriteDump() = does not match: %v", err) } diff --git a/internal/dump/tar.go b/internal/dump/tar.go index c5933d4f8..f8a5b98f1 100644 --- a/internal/dump/tar.go +++ b/internal/dump/tar.go @@ -8,22 +8,16 @@ import ( "path/filepath" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" ) func (d *Dumper) dumpTar(ctx context.Context, ch <-chan *restic.Node) (err error) { - w := tar.NewWriter(d.w) - - defer func() { - if err == nil { - err = w.Close() - err = errors.Wrap(err, "Close") - } - }() + if d.tarWriter == nil { + d.tarWriter = tar.NewWriter(d.w) + } for node := range ch { - if err := d.dumpNodeTar(ctx, node, w); err != nil { + if err := d.dumpNodeTar(ctx, node); err != nil { return err } } @@ -48,7 +42,7 @@ func tarIdentifier(id uint32) int { return int(id) } -func (d *Dumper) dumpNodeTar(ctx context.Context, node *restic.Node, w *tar.Writer) error { +func (d *Dumper) dumpNodeTar(ctx context.Context, node *restic.Node) error { relPath, err := filepath.Rel("/", node.Path) if err != nil { return err @@ -93,11 +87,11 @@ func (d *Dumper) dumpNodeTar(ctx context.Context, node *restic.Node, w *tar.Writ header.Name += "/" } - err = w.WriteHeader(header) + err = d.tarWriter.WriteHeader(header) if err != nil { return fmt.Errorf("writing header for %q: %w", node.Path, err) } - return d.writeNode(ctx, w, node) + return d.writeNode(ctx, d.tarWriter, node) } func parseXattrs(xattrs []restic.ExtendedAttribute) map[string]string { diff --git a/internal/dump/zip.go b/internal/dump/zip.go index d32475770..12942674a 100644 --- a/internal/dump/zip.go +++ b/internal/dump/zip.go @@ -10,24 +10,19 @@ import ( ) func (d *Dumper) dumpZip(ctx context.Context, ch <-chan *restic.Node) (err error) { - w := zip.NewWriter(d.w) - - defer func() { - if err == nil { - err = w.Close() - err = errors.Wrap(err, "Close") - } - }() + if d.zipWriter == nil { + d.zipWriter = zip.NewWriter(d.w) + } for node := range ch { - if err := d.dumpNodeZip(ctx, node, w); err != nil { + if err := d.dumpNodeZip(ctx, node); err != nil { return err } } return nil } -func (d *Dumper) dumpNodeZip(ctx context.Context, node *restic.Node, zw *zip.Writer) error { +func (d *Dumper) dumpNodeZip(ctx context.Context, node *restic.Node) error { relPath, err := filepath.Rel("/", node.Path) if err != nil { return err @@ -44,7 +39,7 @@ func (d *Dumper) dumpNodeZip(ctx context.Context, node *restic.Node, zw *zip.Wri header.Name += "/" } - w, err := zw.CreateHeader(header) + w, err := d.zipWriter.CreateHeader(header) if err != nil { return errors.Wrap(err, "ZipHeader") }