Mirrors/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2025-03-09 00:00:02 +01:00
Code Issues Releases Wiki Activity

rewrite: handling of empty subdirectories, more error checking of exclusive options

Browse source 
cmd_rewrite: more text for the cobra.Command.Long option, more error checking
for incompatible options `--include`/`--exclude` and `--snapshot-summary`.
Adapted change of call to walker.NewSnapshotSizeRewriter for the second parameter.

rewriter.NewSnapshotSizeRewriter: renamed `KeepEmptyDirecoryGlobal` to `RemoveEmptyDirectoryGlobal`
to make logic change clearer and also deal with initialisation.
This commit is contained in:
Winfried Plappert 2025-02-19 07:47:52 +00:00
parent 0a6818cd7a
commit 86e766f24d
2 changed files with 66 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
cmd/restic/cmd_rewrite.go
View file

@ -27,6 +27,16 @@ The "rewrite" command excludes files from existing snapshots. It creates new
snapshots containing the same data as the original ones, but without the files
you specify to exclude. All metadata (time, host, tags) will be preserved.
Alternatively you can use one of the --include variants to only include files
in the new snapshot which you want to preserve. All other files not mayching any
of your --include pattern will not be saved in the new snapshot. Empty subdirectories
however will always be preserved. Totally empty subdirectories (apart from genuine ones)
which have been completey evacuated by not including anything useful
will not be stored in the new snapshot.
If you specify an --include pattern which will not include anything useful, you will still
create a new snapshot if the original snapshot contained one or more empty subdirectories.
The snapshots to rewrite are specified using the --host, --tag and --path options,
or by providing a list of snapshot IDs. Please note that specifying neither any of
these options nor a snapshot ID will cause the command to rewrite all snapshots.
@ -39,9 +49,10 @@ Please note that the --forget option only removes the snapshots and not the actu
data stored in the repository. In order to delete the no longer referenced data,
use the "prune" command.
When rewrite is used with the --snapshot-summary option, a new snapshot is
created containing statistics summary data. Only two fields in the summary will
be non-zero: TotalFilesProcessed and TotalBytesProcessed.
When rewrite is used with the --snapshot-summary option exclusively on a snapshot which
does not contain statistics summary data, a new snapshot is created containing statistics.
All existing data are copied into the new snapshot. Only two fields in the
summary will be non-zero: TotalFilesProcessed and TotalBytesProcessed.
When rewrite is called with one of the --exclude options, TotalFilesProcessed
and TotalBytesProcessed will be updated in the snapshot summary.
@ -105,6 +116,7 @@ type RewriteOptions struct {
Metadata snapshotMetadataArgs
restic.SnapshotFilter
filter.ExcludePatternOptions
filter.IncludePatternOptions
}
func (opts *RewriteOptions) AddFlags(f *pflag.FlagSet) {
@ -116,6 +128,7 @@ func (opts *RewriteOptions) AddFlags(f *pflag.FlagSet) {
initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
opts.ExcludePatternOptions.Add(f)
opts.IncludePatternOptions.Add(f)
}
// rewriteFilterFunc returns the filtered tree ID or an error. If a snapshot summary is returned, the snapshot will
@ -140,25 +153,9 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
var filter rewriteFilterFunc
if len(rejectByNameFuncs) > 0 || opts.SnapshotSummary {
selectByName := func(nodepath string) bool {
for _, reject := range rejectByNameFuncs {
if reject(nodepath) {
return false
}
}
return true
}
rewriteNode := func(node *restic.Node, path string) *restic.Node {
if selectByName(path) {
return node
}
Verbosef("excluding %s\n", path)
return nil
}
rewriter, querySize := walker.NewSnapshotSizeRewriter(rewriteNode)
if len(rejectByNameFuncs) > 0 || len(includeByNameFuncs) > 0 || opts.SnapshotSummary {
rewriteNode := gatherFilters(rejectByNameFuncs, includeByNameFuncs)
rewriter, querySize := walker.NewSnapshotSizeRewriter(rewriteNode, len(includeByNameFuncs) > 0)
filter = func(ctx context.Context, sn *restic.Snapshot) (restic.ID, *restic.SnapshotSummary, error) {
id, err := rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
@ -288,8 +285,14 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
}
func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error {
if !opts.SnapshotSummary && opts.ExcludePatternOptions.Empty() && opts.Metadata.empty() {
return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided")
hasExcludes := !opts.ExcludePatternOptions.Empty()
hasIncludes := !opts.IncludePatternOptions.Empty()
if !opts.SnapshotSummary && !hasExcludes && !hasIncludes && opts.Metadata.empty() {
return errors.Fatal("Nothing to do: no includes/excludes provided and no new metadata provided")
} else if hasExcludes && hasIncludes {
return errors.Fatal("You cannot specify include and exclude options simultaneously!")
} else if (hasExcludes || hasIncludes) && opts.SnapshotSummary {
return errors.Fatal("You cannot specify include or exclude options together with --snapshot-summary!")
}
var (

41
internal/walker/rewriter.go
View file

@ -26,6 +26,7 @@ type RewriteOpts struct {
AllowUnstableSerialization bool
DisableNodeCache bool
RemoveEmptyDirectoryGlobal bool
}
type idMap map[restic.ID]restic.ID
@ -58,7 +59,7 @@ func NewTreeRewriter(opts RewriteOpts) *TreeRewriter {
return rw
}
func NewSnapshotSizeRewriter(rewriteNode NodeRewriteFunc) (*TreeRewriter, QueryRewrittenSizeFunc) {
func NewSnapshotSizeRewriter(rewriteNode NodeRewriteFunc, removeEmptyDirectoryGlobal bool) (*TreeRewriter, QueryRewrittenSizeFunc) {
var count uint
var size uint64
@ -72,6 +73,8 @@ func NewSnapshotSizeRewriter(rewriteNode NodeRewriteFunc) (*TreeRewriter, QueryR
return node
},
DisableNodeCache: true,
// RemoveEmptyDirectoryGlobal = false will force old behaviour for --exclude variants
RemoveEmptyDirectoryGlobal: removeEmptyDirectoryGlobal,
})
ss := func() SnapshotSize {
@ -126,7 +129,36 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, node
continue
}
if node.Type != restic.NodeTypeDir {
path := path.Join(nodepath, node.Name)
node = t.opts.RewriteNode(node, path)
if node == nil {
continue
}
if node.Type != restic.NodeTypeDir {
err = tb.AddNode(node)
if err != nil {
return restic.ID{}, err
}
countInserts++
continue
}
// treat nil as null id
var subtree restic.ID
if node.Subtree != nil {
subtree = *node.Subtree
}
newID, err := t.RewriteTree(ctx, repo, path, subtree)
if err != nil {
return restic.ID{}, err
}
// check for empty subtree condition here
if t.opts.RemoveEmptyDirectoryGlobal && err == nil && newID.IsNull() {
continue
}
node.Subtree = &newID
err = tb.AddNode(node)
if err != nil {
return restic.ID{}, err
@ -146,6 +178,11 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, node
err = tb.AddNode(node)
if err != nil {
return restic.ID{}, err
// check for empty node list
if t.opts.RemoveEmptyDirectoryGlobal && countInserts == 0 {
// current subdirectory is empty - due to no includes: create condition here
return restic.ID{}, nil
}
}