diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 32de08a7e..d8344c5d6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,7 +9,7 @@ on: pull_request: env: - latest_go: "1.19.x" + latest_go: "1.20.x" GO111MODULE: on jobs: @@ -19,18 +19,18 @@ jobs: # list of jobs to run: include: - job_name: Windows - go: 1.19.x + go: 1.20.x os: windows-latest test_smb: true - job_name: macOS - go: 1.19.x + go: 1.20.x os: macOS-latest test_fuse: false test_smb: true - job_name: Linux - go: 1.19.x + go: 1.20.x os: ubuntu-latest test_cloud_backends: true test_fuse: true @@ -38,12 +38,17 @@ jobs: check_changelog: true - job_name: Linux (race) - go: 1.19.x + go: 1.20.x os: ubuntu-latest test_fuse: true test_smb: true test_opts: "-race" + - job_name: Linux + go: 1.19.x + os: ubuntu-latest + test_fuse: true + - job_name: Linux go: 1.18.x os: ubuntu-latest @@ -370,10 +375,10 @@ jobs: uses: golangci/golangci-lint-action@v3 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.49 + version: v1.51 # Optional: show only new issues if it's a pull request. The default value is `false`. only-new-issues: true - args: --verbose --timeout 5m + args: --verbose --timeout 10m # only run golangci-lint for pull requests, otherwise ALL hints get # reported. We need to slowly address all issues until we can enable @@ -418,7 +423,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: push: false context: . diff --git a/changelog/unreleased/issue-3941 b/changelog/unreleased/issue-3941 new file mode 100644 index 000000000..011cd9eaa --- /dev/null +++ b/changelog/unreleased/issue-3941 @@ -0,0 +1,14 @@ +Enhancement: Support `--group-by` for backup parent selection + +The backup command by default selected the parent snapshot based on the hostname +and the backup targets. When the backup path list changed, the backup command +was unable to determine a suitable parent snapshot and had to read all +files again. + +The new `--group-by` option for the backup command allows filtering snapshots +for the parent selection by `host`, `paths` and `tags`. It defaults to +`host,paths` which selects the latest snapshot with hostname and paths matching +those of the backup run. It should be used consistently with `forget --group-by`. + +https://github.com/restic/restic/issues/3941 +https://github.com/restic/restic/pull/4081 diff --git a/changelog/unreleased/issue-4211 b/changelog/unreleased/issue-4211 new file mode 100644 index 000000000..45b7aee83 --- /dev/null +++ b/changelog/unreleased/issue-4211 @@ -0,0 +1,8 @@ +Bugfix: Restic dump now interprets --host and --path correctly + +Restic dump previously confused its --host= and --path= +options: it looked for snapshots with paths called from hosts +called . It now treats the options as intended. + +https://github.com/restic/restic/issues/4211 +https://github.com/restic/restic/pull/4212 diff --git a/changelog/unreleased/pull-4176 b/changelog/unreleased/pull-4176 new file mode 100644 index 000000000..8adf0b40f --- /dev/null +++ b/changelog/unreleased/pull-4176 @@ -0,0 +1,7 @@ +Change: Fix JSON message type of `scan_finished` for the `backup` command + +Restic incorrectly set the `message_type` of the `scan_finished` message to +`status` instead of `verbose_status`. This has now been corrected so that +the messages report the correct type. + +https://github.com/restic/restic/pull/4176 diff --git a/changelog/unreleased/pull-4219 b/changelog/unreleased/pull-4219 new file mode 100644 index 000000000..7d20c3607 --- /dev/null +++ b/changelog/unreleased/pull-4219 @@ -0,0 +1,5 @@ +Enhancement: Upgrade Minio to 7.0.49 + +Upgraded to allow use of the ap-southeast-4 region (Melbourne) + +https://github.com/restic/restic/pull/4219 diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index e59f503db..1244e2ed1 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -89,6 +89,7 @@ type BackupOptions struct { excludePatternOptions Parent string + GroupBy restic.SnapshotGroupByOptions Force bool ExcludeOtherFS bool ExcludeIfPresent []string @@ -120,7 +121,9 @@ func init() { cmdRoot.AddCommand(cmdBackup) f := cmdBackup.Flags() - f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repository that has the same target files/directories, and is not newer than the snapshot time)") + f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time)") + backupOptions.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true} + f.VarP(&backupOptions.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')") f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`) initExcludePatternOptions(f, &backupOptions.excludePatternOptions) @@ -439,7 +442,18 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup if snName == "" { snName = "latest" } - sn, err := restic.FindFilteredSnapshot(ctx, repo.Backend(), repo, []string{opts.Host}, []restic.TagList{}, targets, &timeStampLimit, snName) + f := restic.SnapshotFilter{TimestampLimit: timeStampLimit} + if opts.GroupBy.Host { + f.Hosts = []string{opts.Host} + } + if opts.GroupBy.Path { + f.Paths = targets + } + if opts.GroupBy.Tag { + f.Tags = []restic.TagList{opts.Tags.Flatten()} + } + + sn, err := f.FindLatest(ctx, repo.Backend(), repo, snName) // Snapshot not found is ok if no explicit parent was set if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) { err = nil diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index be9dd5130..d56f7d0c9 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -65,7 +65,7 @@ func init() { // MarkDeprecated only returns an error when the flag is not found panic(err) } - f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use the cache") + f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use existing cache, only read uncached data from repository") } func checkFlags(opts CheckOptions) error { diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 14ab1917a..2f095972a 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -39,7 +39,7 @@ new destination repository using the "init" command. // CopyOptions bundles all options for the copy command. type CopyOptions struct { secondaryRepoOptions - snapshotFilterOptions + restic.SnapshotFilter } var copyOptions CopyOptions @@ -49,7 +49,7 @@ func init() { f := cmdCopy.Flags() initSecondaryRepoOptions(f, ©Options.secondaryRepoOptions, "destination", "to copy snapshots from") - initMultiSnapshotFilterOptions(f, ©Options.snapshotFilterOptions, true) + initMultiSnapshotFilter(f, ©Options.SnapshotFilter, true) } func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error { @@ -108,7 +108,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] } dstSnapshotByOriginal := make(map[restic.ID][]*restic.Snapshot) - for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, opts.Hosts, opts.Tags, opts.Paths, nil) { + for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, &opts.SnapshotFilter, nil) { if sn.Original != nil && !sn.Original.IsNull() { dstSnapshotByOriginal[*sn.Original] = append(dstSnapshotByOriginal[*sn.Original], sn) } @@ -119,8 +119,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] // remember already processed trees across all snapshots visitedTrees := restic.NewIDSet() - for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) { - + for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, &opts.SnapshotFilter, args) { // check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields srcOriginal := *sn.ID() if sn.Original != nil { diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index a480b12f4..cda7b65b9 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -40,7 +40,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er // DumpOptions collects all options for the dump command. type DumpOptions struct { - snapshotFilterOptions + restic.SnapshotFilter Archive string } @@ -50,7 +50,7 @@ func init() { cmdRoot.AddCommand(cmdDump) flags := cmdDump.Flags() - initSingleSnapshotFilterOptions(flags, &dumpOptions.snapshotFilterOptions) + initSingleSnapshotFilter(flags, &dumpOptions.SnapshotFilter) flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"") } @@ -139,7 +139,11 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] } } - sn, err := restic.FindFilteredSnapshot(ctx, repo.Backend(), repo, opts.Paths, opts.Tags, opts.Hosts, nil, snapshotIDString) + sn, err := (&restic.SnapshotFilter{ + Hosts: opts.Hosts, + Paths: opts.Paths, + Tags: opts.Tags, + }).FindLatest(ctx, repo.Backend(), repo, snapshotIDString) if err != nil { return errors.Fatalf("failed to find snapshot: %v", err) } diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 8e5f9b604..e5457c3be 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -51,7 +51,7 @@ type FindOptions struct { PackID, ShowPackID bool CaseInsensitive bool ListLong bool - snapshotFilterOptions + restic.SnapshotFilter } var findOptions FindOptions @@ -70,7 +70,7 @@ func init() { f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern") f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode") - initMultiSnapshotFilterOptions(f, &findOptions.snapshotFilterOptions, true) + initMultiSnapshotFilter(f, &findOptions.SnapshotFilter, true) } type findPattern struct { @@ -618,7 +618,7 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args [] } } - for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, opts.Snapshots) { + for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, opts.Snapshots) { if f.blobIDs != nil || f.treeIDs != nil { if err = f.findIDs(ctx, sn); err != nil && err.Error() != "OK" { return err diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index 472b22b79..fbe4c1c8a 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -52,11 +52,11 @@ type ForgetOptions struct { WithinYearly restic.Duration KeepTags restic.TagLists - snapshotFilterOptions + restic.SnapshotFilter Compact bool // Grouping - GroupBy string + GroupBy restic.SnapshotGroupByOptions DryRun bool Prune bool } @@ -81,7 +81,7 @@ func init() { f.VarP(&forgetOptions.WithinYearly, "keep-within-yearly", "", "keep yearly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot") f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)") - initMultiSnapshotFilterOptions(f, &forgetOptions.snapshotFilterOptions, false) + initMultiSnapshotFilter(f, &forgetOptions.SnapshotFilter, false) f.StringArrayVar(&forgetOptions.Hosts, "hostname", nil, "only consider snapshots with the given `hostname` (can be specified multiple times)") err := f.MarkDeprecated("hostname", "use --host") if err != nil { @@ -90,8 +90,8 @@ func init() { } f.BoolVarP(&forgetOptions.Compact, "compact", "c", false, "use compact output format") - - f.StringVarP(&forgetOptions.GroupBy, "group-by", "g", "host,paths", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')") + forgetOptions.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true} + f.VarP(&forgetOptions.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')") f.BoolVarP(&forgetOptions.DryRun, "dry-run", "n", false, "do not delete anything, just print what would be done") f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed") @@ -126,7 +126,7 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg var snapshots restic.Snapshots removeSnIDs := restic.NewIDSet() - for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { + for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) { snapshots = append(snapshots, sn) } diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 7dd41ab21..aeaa750eb 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -49,7 +49,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er // LsOptions collects all options for the ls command. type LsOptions struct { ListLong bool - snapshotFilterOptions + restic.SnapshotFilter Recursive bool } @@ -59,7 +59,7 @@ func init() { cmdRoot.AddCommand(cmdLs) flags := cmdLs.Flags() - initSingleSnapshotFilterOptions(flags, &lsOptions.snapshotFilterOptions) + initSingleSnapshotFilter(flags, &lsOptions.SnapshotFilter) flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode") flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories") } @@ -210,7 +210,11 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri } } - sn, err := restic.FindFilteredSnapshot(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, nil, args[0]) + sn, err := (&restic.SnapshotFilter{ + Hosts: opts.Hosts, + Paths: opts.Paths, + Tags: opts.Tags, + }).FindLatest(ctx, snapshotLister, repo, args[0]) if err != nil { return err } diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index 7afb30f7c..0501bfe89 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -77,7 +77,7 @@ type MountOptions struct { OwnerRoot bool AllowOther bool NoDefaultPermissions bool - snapshotFilterOptions + restic.SnapshotFilter TimeTemplate string PathTemplates []string } @@ -92,7 +92,7 @@ func init() { mountFlags.BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory") mountFlags.BoolVar(&mountOptions.NoDefaultPermissions, "no-default-permissions", false, "for 'allow-other', ignore Unix permissions and allow users to read all snapshot files") - initMultiSnapshotFilterOptions(mountFlags, &mountOptions.snapshotFilterOptions, true) + initMultiSnapshotFilter(mountFlags, &mountOptions.SnapshotFilter, true) mountFlags.StringArrayVar(&mountOptions.PathTemplates, "path-template", nil, "set `template` for path names (can be specified multiple times)") mountFlags.StringVar(&mountOptions.TimeTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs") @@ -180,9 +180,7 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args cfg := fuse.Config{ OwnerIsRoot: opts.OwnerRoot, - Hosts: opts.Hosts, - Tags: opts.Tags, - Paths: opts.Paths, + Filter: opts.SnapshotFilter, TimeTemplate: opts.TimeTemplate, PathTemplates: opts.PathTemplates, } diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index b70cb52ff..579711662 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -42,7 +42,7 @@ type RestoreOptions struct { Include []string InsensitiveInclude []string Target string - snapshotFilterOptions + restic.SnapshotFilter Sparse bool Verify bool } @@ -59,7 +59,7 @@ func init() { flags.StringArrayVar(&restoreOptions.InsensitiveInclude, "iinclude", nil, "same as `--include` but ignores the casing of filenames") flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to") - initSingleSnapshotFilterOptions(flags, &restoreOptions.snapshotFilterOptions) + initSingleSnapshotFilter(flags, &restoreOptions.SnapshotFilter) flags.BoolVar(&restoreOptions.Sparse, "sparse", false, "restore files as sparse") flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content") } @@ -131,7 +131,11 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, a } } - sn, err := restic.FindFilteredSnapshot(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, nil, snapshotIDString) + sn, err := (&restic.SnapshotFilter{ + Hosts: opts.Hosts, + Paths: opts.Paths, + Tags: opts.Tags, + }).FindLatest(ctx, repo.Backend(), repo, snapshotIDString) if err != nil { return errors.Fatalf("failed to find snapshot: %v", err) } diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index cfe56db87..0d9aa1c8c 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -51,7 +51,7 @@ type RewriteOptions struct { Forget bool DryRun bool - snapshotFilterOptions + restic.SnapshotFilter excludePatternOptions } @@ -64,7 +64,7 @@ func init() { f.BoolVarP(&rewriteOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones") f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done") - initMultiSnapshotFilterOptions(f, &rewriteOptions.snapshotFilterOptions, true) + initMultiSnapshotFilter(f, &rewriteOptions.SnapshotFilter, true) initExcludePatternOptions(f, &rewriteOptions.excludePatternOptions) } @@ -186,7 +186,7 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a } changedCount := 0 - for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, args) { + for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) { Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) changed, err := rewriteSnapshot(ctx, repo, sn, opts) if err != nil { diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index 0bfa4d110..2de8801cb 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -32,11 +32,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er // SnapshotOptions bundles all options for the snapshots command. type SnapshotOptions struct { - snapshotFilterOptions + restic.SnapshotFilter Compact bool Last bool // This option should be removed in favour of Latest. Latest int - GroupBy string + GroupBy restic.SnapshotGroupByOptions } var snapshotOptions SnapshotOptions @@ -45,7 +45,7 @@ func init() { cmdRoot.AddCommand(cmdSnapshots) f := cmdSnapshots.Flags() - initMultiSnapshotFilterOptions(f, &snapshotOptions.snapshotFilterOptions, true) + initMultiSnapshotFilter(f, &snapshotOptions.SnapshotFilter, true) f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact output format") f.BoolVar(&snapshotOptions.Last, "last", false, "only show the last snapshot for each host and path") err := f.MarkDeprecated("last", "use --latest 1") @@ -54,7 +54,7 @@ func init() { panic(err) } f.IntVar(&snapshotOptions.Latest, "latest", 0, "only show the last `n` snapshots for each host and path") - f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "`group` snapshots by host, paths and/or tags, separated by comma") + f.VarP(&snapshotOptions.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma") } func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions, args []string) error { @@ -73,7 +73,7 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions } var snapshots restic.Snapshots - for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { + for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) { snapshots = append(snapshots, sn) } snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy) diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 99d16b932..55ba6f254 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -58,7 +58,7 @@ type StatsOptions struct { // the mode of counting to perform (see consts for available modes) countMode string - snapshotFilterOptions + restic.SnapshotFilter } var statsOptions StatsOptions @@ -67,7 +67,7 @@ func init() { cmdRoot.AddCommand(cmdStats) f := cmdStats.Flags() f.StringVar(&statsOptions.countMode, "mode", countModeRestoreSize, "counting mode: restore-size (default), files-by-contents, blobs-per-file or raw-data") - initMultiSnapshotFilterOptions(f, &statsOptions.snapshotFilterOptions, true) + initMultiSnapshotFilter(f, &statsOptions.SnapshotFilter, true) } func runStats(ctx context.Context, gopts GlobalOptions, args []string) error { @@ -111,7 +111,7 @@ func runStats(ctx context.Context, gopts GlobalOptions, args []string) error { SnapshotsCount: 0, } - for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, statsOptions.Hosts, statsOptions.Tags, statsOptions.Paths, args) { + for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &statsOptions.SnapshotFilter, args) { err = statsWalkSnapshot(ctx, sn, repo, stats) if err != nil { return fmt.Errorf("error walking snapshot: %v", err) diff --git a/cmd/restic/cmd_tag.go b/cmd/restic/cmd_tag.go index 222ddd04a..e5948ea02 100644 --- a/cmd/restic/cmd_tag.go +++ b/cmd/restic/cmd_tag.go @@ -35,7 +35,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er // TagOptions bundles all options for the 'tag' command. type TagOptions struct { - snapshotFilterOptions + restic.SnapshotFilter SetTags restic.TagLists AddTags restic.TagLists RemoveTags restic.TagLists @@ -50,7 +50,7 @@ func init() { tagFlags.Var(&tagOptions.SetTags, "set", "`tags` which will replace the existing tags in the format `tag[,tag,...]` (can be given multiple times)") tagFlags.Var(&tagOptions.AddTags, "add", "`tags` which will be added to the existing tags in the format `tag[,tag,...]` (can be given multiple times)") tagFlags.Var(&tagOptions.RemoveTags, "remove", "`tags` which will be removed from the existing tags in the format `tag[,tag,...]` (can be given multiple times)") - initMultiSnapshotFilterOptions(tagFlags, &tagOptions.snapshotFilterOptions, true) + initMultiSnapshotFilter(tagFlags, &tagOptions.SnapshotFilter, true) } func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, setTags, addTags, removeTags []string) (bool, error) { @@ -119,7 +119,7 @@ func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []st } changeCnt := 0 - for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { + for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) { changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten()) if err != nil { Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err) diff --git a/cmd/restic/find.go b/cmd/restic/find.go index 7b488c7aa..54d3563b1 100644 --- a/cmd/restic/find.go +++ b/cmd/restic/find.go @@ -8,34 +8,28 @@ import ( "github.com/spf13/pflag" ) -type snapshotFilterOptions struct { - Hosts []string - Tags restic.TagLists - Paths []string -} - -// initMultiSnapshotFilterOptions is used for commands that work on multiple snapshots +// initMultiSnapshotFilter is used for commands that work on multiple snapshots // MUST be combined with restic.FindFilteredSnapshots or FindFilteredSnapshots -func initMultiSnapshotFilterOptions(flags *pflag.FlagSet, options *snapshotFilterOptions, addHostShorthand bool) { +func initMultiSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter, addHostShorthand bool) { hostShorthand := "H" if !addHostShorthand { hostShorthand = "" } - flags.StringArrayVarP(&options.Hosts, "host", hostShorthand, nil, "only consider snapshots for this `host` (can be specified multiple times)") - flags.Var(&options.Tags, "tag", "only consider snapshots including `tag[,tag,...]` (can be specified multiple times)") - flags.StringArrayVar(&options.Paths, "path", nil, "only consider snapshots including this (absolute) `path` (can be specified multiple times)") + flags.StringArrayVarP(&filt.Hosts, "host", hostShorthand, nil, "only consider snapshots for this `host` (can be specified multiple times)") + flags.Var(&filt.Tags, "tag", "only consider snapshots including `tag[,tag,...]` (can be specified multiple times)") + flags.StringArrayVar(&filt.Paths, "path", nil, "only consider snapshots including this (absolute) `path` (can be specified multiple times)") } -// initSingleSnapshotFilterOptions is used for commands that work on a single snapshot +// initSingleSnapshotFilter is used for commands that work on a single snapshot // MUST be combined with restic.FindFilteredSnapshot -func initSingleSnapshotFilterOptions(flags *pflag.FlagSet, options *snapshotFilterOptions) { - flags.StringArrayVarP(&options.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times)") - flags.Var(&options.Tags, "tag", "only consider snapshots including `tag[,tag,...]`, when snapshot ID \"latest\" is given (can be specified multiple times)") - flags.StringArrayVar(&options.Paths, "path", nil, "only consider snapshots including this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times)") +func initSingleSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter) { + flags.StringArrayVarP(&filt.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times)") + flags.Var(&filt.Tags, "tag", "only consider snapshots including `tag[,tag,...]`, when snapshot ID \"latest\" is given (can be specified multiple times)") + flags.StringArrayVar(&filt.Paths, "path", nil, "only consider snapshots including this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times)") } // FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. -func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, hosts []string, tags []restic.TagList, paths []string, snapshotIDs []string) <-chan *restic.Snapshot { +func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, f *restic.SnapshotFilter, snapshotIDs []string) <-chan *restic.Snapshot { out := make(chan *restic.Snapshot) go func() { defer close(out) @@ -45,7 +39,7 @@ func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic. return } - err = restic.FindFilteredSnapshots(ctx, be, loader, hosts, tags, paths, snapshotIDs, func(id string, sn *restic.Snapshot, err error) error { + err = f.FindAll(ctx, be, loader, snapshotIDs, func(id string, sn *restic.Snapshot, err error) error { if err != nil { Warnf("Ignoring %q: %v\n", id, err) } else { diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 76c7069ce..0d6079b04 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -113,7 +113,8 @@ func init() { f.StringVarP(&globalOptions.KeyHint, "key-hint", "", "", "`key` ID of key to try decrypting first (default: $RESTIC_KEY_HINT)") f.StringVarP(&globalOptions.PasswordCommand, "password-command", "", "", "shell `command` to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)") f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not output comprehensive progress report") - f.CountVarP(&globalOptions.Verbose, "verbose", "v", "be verbose (specify multiple times or a level using --verbose=`n`, max level/times is 2)") + // use empty paremeter name as `-v, --verbose n` instead of the correct `--verbose=n` is confusing + f.CountVarP(&globalOptions.Verbose, "verbose", "v", "be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2)") f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repository, this allows some operations on read-only repositories") f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it") f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache `directory`. (default: use system default cache directory)") diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 062a5954c..c87722f02 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -106,7 +106,7 @@ func testRunRestore(t testing.TB, opts GlobalOptions, dir string, snapshotID res func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths []string, hosts []string) { opts := RestoreOptions{ Target: dir, - snapshotFilterOptions: snapshotFilterOptions{ + SnapshotFilter: restic.SnapshotFilter{ Hosts: hosts, Paths: paths, }, @@ -2196,7 +2196,7 @@ func TestFindListOnce(t *testing.T) { snapshotIDs := restic.NewIDSet() // specify the two oldest snapshots explicitly and use "latest" to reference the newest one - for sn := range FindFilteredSnapshots(context.TODO(), repo.Backend(), repo, nil, nil, nil, []string{ + for sn := range FindFilteredSnapshots(context.TODO(), repo.Backend(), repo, &restic.SnapshotFilter{}, []string{ secondSnapshot[0].String(), secondSnapshot[1].String()[:8], "latest", diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index aedf3bed8..7484c0e5d 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -90,7 +90,7 @@ command and enter the same password twice: data from a CIFS share is not recommended due to compatibility issues in older Linux kernels. Either use another backend or set the environment variable `GODEBUG` to `asyncpreemptoff=1`. Refer to GitHub issue - `#2659 `_ for further explanations. + :issue:`2659` for further explanations. SFTP **** diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 10fb73815..21b357981 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -139,13 +139,24 @@ File change detection ********************* When restic encounters a file that has already been backed up, whether in the -current backup or a previous one, it makes sure the file's contents are only +current backup or a previous one, it makes sure the file's content is only stored once in the repository. To do so, it normally has to scan the entire -contents of every file. Because this can be very expensive, restic also uses a +content of the file. Because this can be very expensive, restic also uses a change detection rule based on file metadata to determine whether a file is likely unchanged since a previous backup. If it is, the file is not scanned again. +The previous backup snapshot, called "parent" snaphot in restic terminology, +is determined as follows. By default restic groups snapshots by hostname and +backup paths, and then selects the latest snapshot in the group that matches +the current backup. You can change the selection criteria using the +``--group-by`` option, which defaults to ``host,paths``. To select the latest +snapshot with the same paths independent of the hostname, use ``paths``. Or, +to only consider the hostname and tags, use ``host,tags``. Alternatively, it +is possible to manually specify a specific parent snapshot using the +``--parent`` option. Finally, note that one would normally set the +``--group-by`` option for the ``forget`` command to the same value. + Change detection is only performed for regular files (not special files, symlinks or directories) that have the exact same path as they did in a previous backup of the same location. If a file or one of its containing @@ -205,6 +216,7 @@ Combined with ``--verbose``, you can see a list of changes: Would be added to the repository: 25.551 MiB .. _backup-excluding-files: + Excluding Files *************** diff --git a/doc/060_forget.rst b/doc/060_forget.rst index a4205de75..b960ddb14 100644 --- a/doc/060_forget.rst +++ b/doc/060_forget.rst @@ -219,6 +219,8 @@ paths and tags. The policy is then applied to each group of snapshots individual This is a safety feature to prevent accidental removal of unrelated backup sets. To disable grouping and apply the policy to all snapshots regardless of their host, paths and tags, use ``--group-by ''`` (that is, an empty value to ``--group-by``). +Note that one would normally set the ``--group-by`` option for the ``backup`` +command to the same value. Additionally, you can restrict the policy to only process snapshots which have a particular hostname with the ``--host`` parameter, or tags with the ``--tag`` diff --git a/doc/conf.py b/doc/conf.py index 3c0af927b..3fd8dc119 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -106,5 +106,5 @@ html_static_path = ['_static'] htmlhelp_basename = 'resticdoc' extlinks = { - 'issue': ('https://github.com/restic/restic/issues/%s', '#'), + 'issue': ('https://github.com/restic/restic/issues/%s', '#%s'), } diff --git a/doc/design.rst b/doc/design.rst index 3e25a0852..7102585ac 100644 --- a/doc/design.rst +++ b/doc/design.rst @@ -45,10 +45,12 @@ comparing its output to the file name. If the prefix of a filename is unique amongst all the other files in the same directory, the prefix may be used instead of the complete filename. -Apart from the files stored within the ``keys`` directory, all files are -encrypted with AES-256 in counter mode (CTR). The integrity of the -encrypted data is secured by a Poly1305-AES message authentication code -(sometimes also referred to as a "signature"). +Apart from the files stored within the ``keys`` and ``data`` directories, +all files are encrypted with AES-256 in counter mode (CTR). The integrity +of the encrypted data is secured by a Poly1305-AES message authentication +code (sometimes also referred to as a "signature"). +Files in the ``data`` directory ("pack files") consist of multiple parts +which are all independently encrypted and authenticated, see below. In the first 16 bytes of each encrypted file the initialisation vector (IV) is stored. It is followed by the encrypted data and completed by @@ -276,7 +278,7 @@ of a JSON document like the following: }, { "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae", - "type": "tree", + "type": "data", "offset": 38, "length": 112, "uncompressed_length": 511, @@ -298,8 +300,8 @@ example, the Pack ``73d04e61`` contains two data Blobs and one Tree blob, the plaintext hashes are listed afterwards. The ``length`` field corresponds to ``Length(encrypted_blob)`` in the pack file header. Field ``uncompressed_length`` is only present for compressed blobs and -therefore is never present in version 1. It is set to the value of -``Length(blob)``. +therefore is never present in version 1 of the repository format. It is +set to the value of ``Length(blob)``. The field ``supersedes`` lists the storage IDs of index files that have been replaced with the current index file. This happens when index files @@ -410,7 +412,9 @@ and pretty-print the contents of a snapshot file: { "time": "2015-01-02T18:10:50.895208559+01:00", "tree": "2da81727b6585232894cfbb8f8bdab8d1eccd3d8f7c92bc934d62e62e618ffdf", - "dir": "/tmp/testdata", + "paths": [ + "/tmp/testdata" + ], "hostname": "kasimir", "username": "fd0", "uid": 1000, @@ -436,7 +440,9 @@ becomes: { "time": "2015-01-02T18:10:50.895208559+01:00", "tree": "2da81727b6585232894cfbb8f8bdab8d1eccd3d8f7c92bc934d62e62e618ffdf", - "dir": "/tmp/testdata", + "paths": [ + "/tmp/testdata" + ], "hostname": "kasimir", "username": "fd0", "uid": 1000, @@ -495,9 +501,18 @@ the JSON is indented): } A tree contains a list of entries (in the field ``nodes``) which contain -meta data like a name and timestamps. When the entry references a -directory, the field ``subtree`` contains the plain text ID of another -tree object. +meta data like a name and timestamps. Note that there are some specialities of how +this metadata is generated: + +- The name is quoted using `strconv.Quote `__ + before being saved. This handles non-unicode names, but also changes the + representation of names containing ``"`` or ``\``. + +- The filemode saved is the mode defined by `fs.FileMode `__ + masked by ``os.ModePerm | os.ModeType | os.ModeSetuid | os.ModeSetgid | os.ModeSticky`` + +When the entry references a directory, the field ``subtree`` contains the plain text +ID of another tree object. When the command ``restic cat blob`` is used, the plaintext ID is needed to print a tree. The tree referenced above can be dumped as follows: diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index 899a45688..97480db80 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -67,7 +67,7 @@ Usage help is available: -r, --repo repository repository to backup to or restore from (default: $RESTIC_REPOSITORY) --repository-file file file to read the repository location from (default: $RESTIC_REPOSITORY_FILE) --tls-client-cert file path to a file containing PEM encoded TLS client certificate and private key - -v, --verbose n be verbose (specify multiple times or a level using --verbose=n, max level/times is 2) + -v, --verbose be verbose (specify multiple times or a level using --verbose=n, max level/times is 2) Use "restic [command] --help" for more information about a command. @@ -142,7 +142,7 @@ command: -r, --repo repository repository to backup to or restore from (default: $RESTIC_REPOSITORY) --repository-file file file to read the repository location from (default: $RESTIC_REPOSITORY_FILE) --tls-client-cert file path to a file containing PEM encoded TLS client certificate and private key - -v, --verbose n be verbose (specify multiple times or a level using --verbose=n, max level/times is 2) + -v, --verbose be verbose (specify multiple times or a level using --verbose=n, max level/times is 2) Subcommands that support showing progress information such as ``backup``, ``check`` and ``prune`` will do so unless the quiet flag ``-q`` or diff --git a/go.mod b/go.mod index 76beb6846..3271cf230 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/restic/restic require ( cloud.google.com/go/storage v1.29.0 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 github.com/anacrolix/fuse v0.2.0 github.com/cenkalti/backoff/v4 v4.2.0 @@ -14,9 +14,9 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/hirochachacha/go-smb2 v1.1.0 github.com/juju/ratelimit v1.0.2 - github.com/klauspost/compress v1.15.15 + github.com/klauspost/compress v1.16.0 github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5 - github.com/minio/minio-go/v7 v7.0.47 + github.com/minio/minio-go/v7 v7.0.49 github.com/minio/sha256-simd v1.0.0 github.com/ncw/swift/v2 v2.0.1 github.com/pkg/errors v0.9.1 @@ -26,31 +26,31 @@ require ( github.com/restic/chunker v0.4.0 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 - golang.org/x/crypto v0.5.0 - golang.org/x/net v0.5.0 - golang.org/x/oauth2 v0.4.0 + golang.org/x/crypto v0.7.0 + golang.org/x/net v0.8.0 + golang.org/x/oauth2 v0.5.0 golang.org/x/sync v0.1.0 - golang.org/x/sys v0.4.0 - golang.org/x/term v0.4.0 - golang.org/x/text v0.6.0 - google.golang.org/api v0.108.0 + golang.org/x/sys v0.6.0 + golang.org/x/term v0.6.0 + golang.org/x/text v0.8.0 + google.golang.org/api v0.111.0 ) require ( cloud.google.com/go v0.108.0 // indirect - cloud.google.com/go/compute v1.15.1 // indirect + cloud.google.com/go/compute v1.18.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.10.0 // indirect + cloud.google.com/go/iam v0.11.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/dnaeon/go-vcr v1.2.0 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/fgprof v0.9.3 // indirect github.com/geoffgarside/ber v1.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/googleapis/gax-go/v2 v2.7.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -65,8 +65,8 @@ require ( go.opencensus.io v0.24.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect - google.golang.org/grpc v1.52.0 // indirect + google.golang.org/genproto v0.0.0-20230223222841-637eb2293923 // indirect + google.golang.org/grpc v1.53.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 3561bbdf8..878707452 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,17 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.108.0 h1:xntQwnfn8oHGX0crLVinvHM+AhXvi3QHQIEcX/2hiWk= cloud.google.com/go v0.108.0/go.mod h1:lNUfQqusBJp0bgAg6qrHgYFYbTB+dOiob1itwnlD33Q= -cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE= -cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= +cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= +cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/iam v0.10.0 h1:fpP/gByFs6US1ma53v7VxhvbJpO2Aapng6wabJ99MuI= -cloud.google.com/go/iam v0.10.0/go.mod h1:nXAECrMt2qHpF6RZUZseteD6QyanL68reN4OXPw0UWM= +cloud.google.com/go/iam v0.11.0 h1:kwCWfKwB6ePZoZnGLwrd3B6Ru/agoHANTUBWpVNIdnM= +cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.1 h1:gVXuXcWd1i4C2Ruxe321aU+IKGaStvGB/S90PUPB/W8= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.1/go.mod h1:DffdKW9RFqa5VgmsjUOsS7UE7eiA5iAvYUs63bhKQ0M= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= @@ -39,8 +39,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/elithrar/simple-scrypt v1.3.0 h1:KIlOlxdoQf9JWKl5lMAJ28SY2URB0XTRDn2TckyzAZg= @@ -91,8 +91,8 @@ github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b/go.mod h1:dDKJzRmX4S3 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.1 h1:RY7tHKZcRlk788d5WSo/e83gOyyy742E8GSs771ySpg= -github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= @@ -107,8 +107,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI= github.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= -github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= -github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= +github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= +github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= @@ -120,8 +120,8 @@ github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5/go.mod h1:4FCXMUWo9 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.47 h1:sLiuCKGSIcn/MI6lREmTzX91DX/oRau4ia0j6e6eOSs= -github.com/minio/minio-go/v7 v7.0.47/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw= +github.com/minio/minio-go/v7 v7.0.49 h1:dE5DfOtnXMXCjr/HWI6zN9vCrY6Sv666qhhiwUMvGV4= +github.com/minio/minio-go/v7 v7.0.49/go.mod h1:UI34MvQEiob3Cf/gGExGMmzugkM/tNgbFypNDy5LMVc= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -177,8 +177,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -194,11 +194,11 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= -golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= +golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -219,17 +219,17 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= -golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -242,8 +242,8 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/api v0.108.0 h1:WVBc/faN0DkKtR43Q/7+tPny9ZoLZdIiAyG5Q9vFClg= -google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.111.0 h1:bwKi+z2BsdwYFRKrqwutM+axAlYLz83gt5pDSXCJT+0= +google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= @@ -251,15 +251,15 @@ google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230223222841-637eb2293923 h1:znp6mq/drrY+6khTAlJUDNFFcDGV2ENLYKpMq8SyCds= +google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.52.0 h1:kd48UiU7EHsV4rnLyOJRuP/Il/UHE7gdDAQ+SZI7nZk= -google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= +google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= +google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/internal/fuse/root.go b/internal/fuse/root.go index fc8841964..ab6116f0d 100644 --- a/internal/fuse/root.go +++ b/internal/fuse/root.go @@ -16,9 +16,7 @@ import ( // Config holds settings for the fuse mount. type Config struct { OwnerIsRoot bool - Hosts []string - Tags []restic.TagList - Paths []string + Filter restic.SnapshotFilter TimeTemplate string PathTemplates []string } diff --git a/internal/fuse/snapshots_dirstruct.go b/internal/fuse/snapshots_dirstruct.go index f8e66d076..3080d4de8 100644 --- a/internal/fuse/snapshots_dirstruct.go +++ b/internal/fuse/snapshots_dirstruct.go @@ -295,7 +295,7 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error { } var snapshots restic.Snapshots - err := restic.FindFilteredSnapshots(ctx, d.root.repo.Backend(), d.root.repo, d.root.cfg.Hosts, d.root.cfg.Tags, d.root.cfg.Paths, nil, func(id string, sn *restic.Snapshot, err error) error { + err := d.root.cfg.Filter.FindAll(ctx, d.root.repo.Backend(), d.root.repo, nil, func(id string, sn *restic.Snapshot, err error) error { if sn != nil { snapshots = append(snapshots, sn) } diff --git a/internal/restic/snapshot_find.go b/internal/restic/snapshot_find.go index 4f8231a7f..4d4bb4957 100644 --- a/internal/restic/snapshot_find.go +++ b/internal/restic/snapshot_find.go @@ -12,13 +12,32 @@ import ( // ErrNoSnapshotFound is returned when no snapshot for the given criteria could be found. var ErrNoSnapshotFound = errors.New("no snapshot found") -// findLatestSnapshot finds latest snapshot with optional target/directory, tags, hostname, and timestamp filters. -func findLatestSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, hosts []string, - tags []TagList, paths []string, timeStampLimit *time.Time) (*Snapshot, error) { +// A SnapshotFilter denotes a set of snapshots based on hosts, tags and paths. +type SnapshotFilter struct { + _ struct{} // Force naming fields in literals. + + Hosts []string + Tags TagLists + Paths []string + // Match snapshots from before this timestamp. Zero for no limit. + TimestampLimit time.Time +} + +func (f *SnapshotFilter) empty() bool { + return len(f.Hosts)+len(f.Tags)+len(f.Paths) == 0 +} + +func (f *SnapshotFilter) matches(sn *Snapshot) bool { + return sn.HasHostname(f.Hosts) && sn.HasTagList(f.Tags) && sn.HasPaths(f.Paths) +} + +// findLatest finds the latest snapshot with optional target/directory, +// tags, hostname, and timestamp filters. +func (f *SnapshotFilter) findLatest(ctx context.Context, be Lister, loader LoaderUnpacked) (*Snapshot, error) { var err error - absTargets := make([]string, 0, len(paths)) - for _, target := range paths { + absTargets := make([]string, 0, len(f.Paths)) + for _, target := range f.Paths { if !filepath.IsAbs(target) { target, err = filepath.Abs(target) if err != nil { @@ -35,7 +54,7 @@ func findLatestSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, h return errors.Errorf("Error loading snapshot %v: %v", id.Str(), err) } - if timeStampLimit != nil && snapshot.Time.After(*timeStampLimit) { + if !f.TimestampLimit.IsZero() && snapshot.Time.After(f.TimestampLimit) { return nil } @@ -43,15 +62,7 @@ func findLatestSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, h return nil } - if !snapshot.HasHostname(hosts) { - return nil - } - - if !snapshot.HasTagList(tags) { - return nil - } - - if !snapshot.HasPaths(absTargets) { + if !f.matches(snapshot) { return nil } @@ -85,12 +96,14 @@ func FindSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, s strin return LoadSnapshot(ctx, loader, id) } -// FindFilteredSnapshot returns either the latests from a filtered list of all snapshots or a snapshot specified by `snapshotID`. -func FindFilteredSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, hosts []string, tags []TagList, paths []string, timeStampLimit *time.Time, snapshotID string) (*Snapshot, error) { +// FindLatest returns either the latest of a filtered list of all snapshots +// or a snapshot specified by `snapshotID`. +func (f *SnapshotFilter) FindLatest(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotID string) (*Snapshot, error) { if snapshotID == "latest" { - sn, err := findLatestSnapshot(ctx, be, loader, hosts, tags, paths, timeStampLimit) + sn, err := f.findLatest(ctx, be, loader) if err == ErrNoSnapshotFound { - err = fmt.Errorf("snapshot filter (Paths:%v Tags:%v Hosts:%v): %w", paths, tags, hosts, err) + err = fmt.Errorf("snapshot filter (Paths:%v Tags:%v Hosts:%v): %w", + f.Paths, f.Tags, f.Hosts, err) } return sn, err } @@ -99,8 +112,8 @@ func FindFilteredSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, type SnapshotFindCb func(string, *Snapshot, error) error -// FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. -func FindFilteredSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked, hosts []string, tags []TagList, paths []string, snapshotIDs []string, fn SnapshotFindCb) error { +// FindAll yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. +func (f *SnapshotFilter) FindAll(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotIDs []string, fn SnapshotFindCb) error { if len(snapshotIDs) != 0 { var err error usedFilter := false @@ -116,9 +129,10 @@ func FindFilteredSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked usedFilter = true - sn, err = findLatestSnapshot(ctx, be, loader, hosts, tags, paths, nil) + sn, err = f.findLatest(ctx, be, loader) if err == ErrNoSnapshotFound { - err = errors.Errorf("no snapshot matched given filter (Paths:%v Tags:%v Hosts:%v)", paths, tags, hosts) + err = errors.Errorf("no snapshot matched given filter (Paths:%v Tags:%v Hosts:%v)", + f.Paths, f.Tags, f.Hosts) } if sn != nil { ids.Insert(*sn.ID()) @@ -141,18 +155,14 @@ func FindFilteredSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked } // Give the user some indication their filters are not used. - if !usedFilter && (len(hosts) != 0 || len(tags) != 0 || len(paths) != 0) { + if !usedFilter && !f.empty() { return fn("filters", nil, errors.Errorf("explicit snapshot ids are given")) } return nil } return ForAllSnapshots(ctx, be, loader, nil, func(id ID, sn *Snapshot, err error) error { - if err != nil { - return fn(id.String(), sn, err) - } - - if !sn.HasHostname(hosts) || !sn.HasTagList(tags) || !sn.HasPaths(paths) { + if err == nil && !f.matches(sn) { return nil } diff --git a/internal/restic/snapshot_find_test.go b/internal/restic/snapshot_find_test.go index 3c587dde1..d098b5224 100644 --- a/internal/restic/snapshot_find_test.go +++ b/internal/restic/snapshot_find_test.go @@ -14,13 +14,14 @@ func TestFindLatestSnapshot(t *testing.T) { restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1, 0) latestSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1, 0) - sn, err := restic.FindFilteredSnapshot(context.TODO(), repo.Backend(), repo, []string{"foo"}, []restic.TagList{}, []string{}, nil, "latest") + f := restic.SnapshotFilter{Hosts: []string{"foo"}} + sn, err := f.FindLatest(context.TODO(), repo.Backend(), repo, "latest") if err != nil { - t.Fatalf("FindLatestSnapshot returned error: %v", err) + t.Fatalf("FindLatest returned error: %v", err) } if *sn.ID() != *latestSnapshot.ID() { - t.Errorf("FindLatestSnapshot returned wrong snapshot ID: %v", *sn.ID()) + t.Errorf("FindLatest returned wrong snapshot ID: %v", *sn.ID()) } } @@ -30,14 +31,15 @@ func TestFindLatestSnapshotWithMaxTimestamp(t *testing.T) { desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1, 0) restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1, 0) - maxTimestamp := parseTimeUTC("2018-08-08 08:08:08") - - sn, err := restic.FindFilteredSnapshot(context.TODO(), repo.Backend(), repo, []string{"foo"}, []restic.TagList{}, []string{}, &maxTimestamp, "latest") + sn, err := (&restic.SnapshotFilter{ + Hosts: []string{"foo"}, + TimestampLimit: parseTimeUTC("2018-08-08 08:08:08"), + }).FindLatest(context.TODO(), repo.Backend(), repo, "latest") if err != nil { - t.Fatalf("FindLatestSnapshot returned error: %v", err) + t.Fatalf("FindLatest returned error: %v", err) } if *sn.ID() != *desiredSnapshot.ID() { - t.Errorf("FindLatestSnapshot returned wrong snapshot ID: %v", *sn.ID()) + t.Errorf("FindLatest returned wrong snapshot ID: %v", *sn.ID()) } } diff --git a/internal/restic/snapshot_group.go b/internal/restic/snapshot_group.go index c8b1a5faa..c3f3307f6 100644 --- a/internal/restic/snapshot_group.go +++ b/internal/restic/snapshot_group.go @@ -8,6 +8,57 @@ import ( "github.com/restic/restic/internal/errors" ) +type SnapshotGroupByOptions struct { + Tag bool + Host bool + Path bool +} + +func splitSnapshotGroupBy(s string) (SnapshotGroupByOptions, error) { + var l SnapshotGroupByOptions + for _, option := range strings.Split(s, ",") { + switch option { + case "host", "hosts": + l.Host = true + case "path", "paths": + l.Path = true + case "tag", "tags": + l.Tag = true + case "": + default: + return SnapshotGroupByOptions{}, errors.Fatal("unknown grouping option: '" + option + "'") + } + } + return l, nil +} + +func (l SnapshotGroupByOptions) String() string { + var parts []string + if l.Host { + parts = append(parts, "host") + } + if l.Path { + parts = append(parts, "paths") + } + if l.Tag { + parts = append(parts, "tags") + } + return strings.Join(parts, ",") +} + +func (l *SnapshotGroupByOptions) Set(s string) error { + parts, err := splitSnapshotGroupBy(s) + if err != nil { + return err + } + *l = parts + return nil +} + +func (l *SnapshotGroupByOptions) Type() string { + return "group" +} + // SnapshotGroupKey is the structure for identifying groups in a grouped // snapshot list. This is used by GroupSnapshots() type SnapshotGroupKey struct { @@ -18,43 +69,24 @@ type SnapshotGroupKey struct { // GroupSnapshots takes a list of snapshots and a grouping criteria and creates // a group list of snapshots. -func GroupSnapshots(snapshots Snapshots, options string) (map[string]Snapshots, bool, error) { +func GroupSnapshots(snapshots Snapshots, groupBy SnapshotGroupByOptions) (map[string]Snapshots, bool, error) { // group by hostname and dirs snapshotGroups := make(map[string]Snapshots) - var GroupByTag bool - var GroupByHost bool - var GroupByPath bool - GroupOptionList := strings.Split(options, ",") - - for _, option := range GroupOptionList { - switch option { - case "host", "hosts": - GroupByHost = true - case "path", "paths": - GroupByPath = true - case "tag", "tags": - GroupByTag = true - case "": - default: - return nil, false, errors.Fatal("unknown grouping option: '" + option + "'") - } - } - for _, sn := range snapshots { // Determining grouping-keys var tags []string var hostname string var paths []string - if GroupByTag { + if groupBy.Tag { tags = sn.Tags sort.Strings(tags) } - if GroupByHost { + if groupBy.Host { hostname = sn.Hostname } - if GroupByPath { + if groupBy.Path { paths = sn.Paths } @@ -70,5 +102,5 @@ func GroupSnapshots(snapshots Snapshots, options string) (map[string]Snapshots, snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn) } - return snapshotGroups, GroupByTag || GroupByHost || GroupByPath, nil + return snapshotGroups, groupBy.Tag || groupBy.Host || groupBy.Path, nil } diff --git a/internal/restic/snapshot_group_test.go b/internal/restic/snapshot_group_test.go new file mode 100644 index 000000000..78ac99ab1 --- /dev/null +++ b/internal/restic/snapshot_group_test.go @@ -0,0 +1,50 @@ +package restic_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" +) + +func TestGroupByOptions(t *testing.T) { + for _, exp := range []struct { + from string + opts restic.SnapshotGroupByOptions + normalized string + }{ + { + from: "", + opts: restic.SnapshotGroupByOptions{}, + normalized: "", + }, + { + from: "host,paths", + opts: restic.SnapshotGroupByOptions{Host: true, Path: true}, + normalized: "host,paths", + }, + { + from: "host,path,tag", + opts: restic.SnapshotGroupByOptions{Host: true, Path: true, Tag: true}, + normalized: "host,paths,tags", + }, + { + from: "hosts,paths,tags", + opts: restic.SnapshotGroupByOptions{Host: true, Path: true, Tag: true}, + normalized: "host,paths,tags", + }, + } { + var opts restic.SnapshotGroupByOptions + test.OK(t, opts.Set(exp.from)) + if !cmp.Equal(opts, exp.opts) { + t.Errorf("unexpeted opts %s", cmp.Diff(opts, exp.opts)) + } + test.Equals(t, opts.String(), exp.normalized) + } + + var opts restic.SnapshotGroupByOptions + err := opts.Set("tags,invalid") + test.Assert(t, err != nil, "missing error on invalid tags") + test.Assert(t, !opts.Host && !opts.Path && !opts.Tag, "unexpected opts %s %s %s", opts.Host, opts.Path, opts.Tag) +} diff --git a/internal/ui/backup/json.go b/internal/ui/backup/json.go index 85076b3bb..e736d0118 100644 --- a/internal/ui/backup/json.go +++ b/internal/ui/backup/json.go @@ -164,7 +164,7 @@ func (b *JSONProgress) CompleteItem(messageType, item string, previous, current func (b *JSONProgress) ReportTotal(item string, start time.Time, s archiver.ScanStats) { if b.v >= 2 { b.print(verboseUpdate{ - MessageType: "status", + MessageType: "verbose_status", Action: "scan_finished", Duration: time.Since(start).Seconds(), DataSize: s.Bytes,