mirror of
https://github.com/restic/restic.git
synced 2025-03-09 00:00:02 +01:00
rewrite: handling of empty subdirectories, more error checking of exclusive options
cmd_rewrite: more text for the cobra.Command.Long option, more error checking for incompatible options `--include`/`--exclude` and `--snapshot-summary`. Adapted change of call to walker.NewSnapshotSizeRewriter for the second parameter. rewriter.NewSnapshotSizeRewriter: renamed `KeepEmptyDirecoryGlobal` to `RemoveEmptyDirectoryGlobal` to make logic change clearer and also deal with initialisation.
This commit is contained in:
parent
0a6818cd7a
commit
86e766f24d
2 changed files with 66 additions and 26 deletions
|
@ -27,6 +27,16 @@ The "rewrite" command excludes files from existing snapshots. It creates new
|
|||
snapshots containing the same data as the original ones, but without the files
|
||||
you specify to exclude. All metadata (time, host, tags) will be preserved.
|
||||
|
||||
Alternatively you can use one of the --include variants to only include files
|
||||
in the new snapshot which you want to preserve. All other files not mayching any
|
||||
of your --include pattern will not be saved in the new snapshot. Empty subdirectories
|
||||
however will always be preserved. Totally empty subdirectories (apart from genuine ones)
|
||||
which have been completey evacuated by not including anything useful
|
||||
will not be stored in the new snapshot.
|
||||
|
||||
If you specify an --include pattern which will not include anything useful, you will still
|
||||
create a new snapshot if the original snapshot contained one or more empty subdirectories.
|
||||
|
||||
The snapshots to rewrite are specified using the --host, --tag and --path options,
|
||||
or by providing a list of snapshot IDs. Please note that specifying neither any of
|
||||
these options nor a snapshot ID will cause the command to rewrite all snapshots.
|
||||
|
@ -39,9 +49,10 @@ Please note that the --forget option only removes the snapshots and not the actu
|
|||
data stored in the repository. In order to delete the no longer referenced data,
|
||||
use the "prune" command.
|
||||
|
||||
When rewrite is used with the --snapshot-summary option, a new snapshot is
|
||||
created containing statistics summary data. Only two fields in the summary will
|
||||
be non-zero: TotalFilesProcessed and TotalBytesProcessed.
|
||||
When rewrite is used with the --snapshot-summary option exclusively on a snapshot which
|
||||
does not contain statistics summary data, a new snapshot is created containing statistics.
|
||||
All existing data are copied into the new snapshot. Only two fields in the
|
||||
summary will be non-zero: TotalFilesProcessed and TotalBytesProcessed.
|
||||
|
||||
When rewrite is called with one of the --exclude options, TotalFilesProcessed
|
||||
and TotalBytesProcessed will be updated in the snapshot summary.
|
||||
|
@ -105,6 +116,7 @@ type RewriteOptions struct {
|
|||
Metadata snapshotMetadataArgs
|
||||
restic.SnapshotFilter
|
||||
filter.ExcludePatternOptions
|
||||
filter.IncludePatternOptions
|
||||
}
|
||||
|
||||
func (opts *RewriteOptions) AddFlags(f *pflag.FlagSet) {
|
||||
|
@ -116,6 +128,7 @@ func (opts *RewriteOptions) AddFlags(f *pflag.FlagSet) {
|
|||
|
||||
initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
|
||||
opts.ExcludePatternOptions.Add(f)
|
||||
opts.IncludePatternOptions.Add(f)
|
||||
}
|
||||
|
||||
// rewriteFilterFunc returns the filtered tree ID or an error. If a snapshot summary is returned, the snapshot will
|
||||
|
@ -140,25 +153,9 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
|
|||
|
||||
var filter rewriteFilterFunc
|
||||
|
||||
if len(rejectByNameFuncs) > 0 || opts.SnapshotSummary {
|
||||
selectByName := func(nodepath string) bool {
|
||||
for _, reject := range rejectByNameFuncs {
|
||||
if reject(nodepath) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
rewriteNode := func(node *restic.Node, path string) *restic.Node {
|
||||
if selectByName(path) {
|
||||
return node
|
||||
}
|
||||
Verbosef("excluding %s\n", path)
|
||||
return nil
|
||||
}
|
||||
|
||||
rewriter, querySize := walker.NewSnapshotSizeRewriter(rewriteNode)
|
||||
if len(rejectByNameFuncs) > 0 || len(includeByNameFuncs) > 0 || opts.SnapshotSummary {
|
||||
rewriteNode := gatherFilters(rejectByNameFuncs, includeByNameFuncs)
|
||||
rewriter, querySize := walker.NewSnapshotSizeRewriter(rewriteNode, len(includeByNameFuncs) > 0)
|
||||
|
||||
filter = func(ctx context.Context, sn *restic.Snapshot) (restic.ID, *restic.SnapshotSummary, error) {
|
||||
id, err := rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
|
||||
|
@ -288,8 +285,14 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
|
|||
}
|
||||
|
||||
func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error {
|
||||
if !opts.SnapshotSummary && opts.ExcludePatternOptions.Empty() && opts.Metadata.empty() {
|
||||
return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided")
|
||||
hasExcludes := !opts.ExcludePatternOptions.Empty()
|
||||
hasIncludes := !opts.IncludePatternOptions.Empty()
|
||||
if !opts.SnapshotSummary && !hasExcludes && !hasIncludes && opts.Metadata.empty() {
|
||||
return errors.Fatal("Nothing to do: no includes/excludes provided and no new metadata provided")
|
||||
} else if hasExcludes && hasIncludes {
|
||||
return errors.Fatal("You cannot specify include and exclude options simultaneously!")
|
||||
} else if (hasExcludes || hasIncludes) && opts.SnapshotSummary {
|
||||
return errors.Fatal("You cannot specify include or exclude options together with --snapshot-summary!")
|
||||
}
|
||||
|
||||
var (
|
||||
|
|
|
@ -26,6 +26,7 @@ type RewriteOpts struct {
|
|||
|
||||
AllowUnstableSerialization bool
|
||||
DisableNodeCache bool
|
||||
RemoveEmptyDirectoryGlobal bool
|
||||
}
|
||||
|
||||
type idMap map[restic.ID]restic.ID
|
||||
|
@ -58,7 +59,7 @@ func NewTreeRewriter(opts RewriteOpts) *TreeRewriter {
|
|||
return rw
|
||||
}
|
||||
|
||||
func NewSnapshotSizeRewriter(rewriteNode NodeRewriteFunc) (*TreeRewriter, QueryRewrittenSizeFunc) {
|
||||
func NewSnapshotSizeRewriter(rewriteNode NodeRewriteFunc, removeEmptyDirectoryGlobal bool) (*TreeRewriter, QueryRewrittenSizeFunc) {
|
||||
var count uint
|
||||
var size uint64
|
||||
|
||||
|
@ -72,6 +73,8 @@ func NewSnapshotSizeRewriter(rewriteNode NodeRewriteFunc) (*TreeRewriter, QueryR
|
|||
return node
|
||||
},
|
||||
DisableNodeCache: true,
|
||||
// RemoveEmptyDirectoryGlobal = false will force old behaviour for --exclude variants
|
||||
RemoveEmptyDirectoryGlobal: removeEmptyDirectoryGlobal,
|
||||
})
|
||||
|
||||
ss := func() SnapshotSize {
|
||||
|
@ -126,7 +129,36 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, node
|
|||
continue
|
||||
}
|
||||
|
||||
if node.Type != restic.NodeTypeDir {
|
||||
path := path.Join(nodepath, node.Name)
|
||||
node = t.opts.RewriteNode(node, path)
|
||||
if node == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if node.Type != restic.NodeTypeDir {
|
||||
err = tb.AddNode(node)
|
||||
if err != nil {
|
||||
return restic.ID{}, err
|
||||
}
|
||||
countInserts++
|
||||
continue
|
||||
}
|
||||
// treat nil as null id
|
||||
var subtree restic.ID
|
||||
if node.Subtree != nil {
|
||||
subtree = *node.Subtree
|
||||
}
|
||||
newID, err := t.RewriteTree(ctx, repo, path, subtree)
|
||||
if err != nil {
|
||||
return restic.ID{}, err
|
||||
}
|
||||
|
||||
// check for empty subtree condition here
|
||||
if t.opts.RemoveEmptyDirectoryGlobal && err == nil && newID.IsNull() {
|
||||
continue
|
||||
}
|
||||
|
||||
node.Subtree = &newID
|
||||
err = tb.AddNode(node)
|
||||
if err != nil {
|
||||
return restic.ID{}, err
|
||||
|
@ -146,6 +178,11 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, node
|
|||
err = tb.AddNode(node)
|
||||
if err != nil {
|
||||
return restic.ID{}, err
|
||||
|
||||
// check for empty node list
|
||||
if t.opts.RemoveEmptyDirectoryGlobal && countInserts == 0 {
|
||||
// current subdirectory is empty - due to no includes: create condition here
|
||||
return restic.ID{}, nil
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Add table
Reference in a new issue