diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d608a8244..07f6b705b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,10 +4,10 @@ updates: - package-ecosystem: "gomod" directory: "/" # Location of package manifests schedule: - interval: "weekly" + interval: "monthly" # Dependencies listed in .github/workflows/*.yml - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "weekly" + interval: "monthly" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 49982aa8a..9d327837b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -64,7 +64,7 @@ jobs: steps: - name: Set up Go ${{ matrix.go }} - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: ${{ matrix.go }} @@ -340,7 +340,7 @@ jobs: steps: - name: Set up Go ${{ env.latest_go }} - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: ${{ env.latest_go }} @@ -365,7 +365,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Go ${{ env.latest_go }} - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: ${{ env.latest_go }} diff --git a/.gitignore b/.gitignore index 812d314b6..b7201c26b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /restic +/restic.exe /.vagrant /.vscode diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e5635404..a502d49e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,92 @@ +Changelog for restic 0.15.2 (2023-04-24) +======================================= + +The following sections list the changes in restic 0.15.2 relevant to +restic users. The changes are ordered by importance. + +Summary +------- + + * Sec #4275: Update golang.org/x/net to address CVE-2022-41723 + * Fix #2260: Sanitize filenames printed by `backup` during processing + * Fix #4211: Make `dump` interpret `--host` and `--path` correctly + * Fix #4239: Correct number of blocks reported in mount point + * Fix #4253: Minimize risk of spurious filesystem loops with `mount` + * Enh #4180: Add release binaries for riscv64 architecture on Linux + * Enh #4219: Upgrade Minio to version 7.0.49 + +Details +------- + + * Security #4275: Update golang.org/x/net to address CVE-2022-41723 + + https://github.com/restic/restic/issues/4275 + https://github.com/restic/restic/pull/4213 + + * Bugfix #2260: Sanitize filenames printed by `backup` during processing + + The `backup` command would previously not sanitize the filenames it printed during + processing, potentially causing newlines or terminal control characters to mangle the + status output or even change the state of a terminal. + + Filenames are now checked and quoted if they contain non-printable or non-Unicode + characters. + + https://github.com/restic/restic/issues/2260 + https://github.com/restic/restic/issues/4191 + https://github.com/restic/restic/pull/4192 + + * Bugfix #4211: Make `dump` interpret `--host` and `--path` correctly + + A regression in restic 0.15.0 caused `dump` to confuse its `--host=` and + `--path=` options: it looked for snapshots with paths called `` from hosts + called ``. It now treats the options as intended. + + https://github.com/restic/restic/issues/4211 + https://github.com/restic/restic/pull/4212 + + * Bugfix #4239: Correct number of blocks reported in mount point + + Restic mount points reported an incorrect number of 512-byte (POSIX standard) blocks for + files and links due to a rounding bug. In particular, empty files were reported as taking one + block instead of zero. + + The rounding is now fixed: the number of blocks reported is the file size (or link target size) + divided by 512 and rounded up to a whole number. + + https://github.com/restic/restic/issues/4239 + https://github.com/restic/restic/pull/4240 + + * Bugfix #4253: Minimize risk of spurious filesystem loops with `mount` + + When a backup contains a directory that has the same name as its parent, say `a/b/b`, and the GNU + `find` command was run on this backup in a restic mount, `find` would refuse to traverse the + lowest `b` directory, instead printing `File system loop detected`. This was due to the way the + restic mount command generates inode numbers for directories in the mount point. + + The rule for generating these inode numbers was changed in 0.15.0. It has now been changed again + to avoid this issue. A perfect rule does not exist, but the probability of this behavior + occurring is now extremely small. + + When it does occur, the mount point is not broken, and scripts that traverse the mount point + should work as long as they don't rely on inode numbers for detecting filesystem loops. + + https://github.com/restic/restic/issues/4253 + https://github.com/restic/restic/pull/4255 + + * Enhancement #4180: Add release binaries for riscv64 architecture on Linux + + Builds for the `riscv64` architecture on Linux are now included in the release binaries. + + https://github.com/restic/restic/pull/4180 + + * Enhancement #4219: Upgrade Minio to version 7.0.49 + + The upgraded version now allows use of the `ap-southeast-4` region (Melbourne). + + https://github.com/restic/restic/pull/4219 + + Changelog for restic 0.15.1 (2023-01-30) ======================================= diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4b4be0757..36a7c0695 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,6 +58,16 @@ Please be aware that the debug log file will contain potentially sensitive things like file and directory names, so please either redact it before uploading it somewhere or post only the parts that are really relevant. +If restic gets stuck, please also include a stacktrace in the description. +On non-Windows systems, you can send a SIGQUIT signal to restic or press +`Ctrl-\` to achieve the same result. This causes restic to print a stacktrace +and then exit immediatelly. This will not damage your repository, however, +it might be necessary to manually clean up stale lock files using +`restic unlock`. + +On Windows, please set the environment variable `RESTIC_DEBUG_STACKTRACE_SIGINT` +to `true` and press `Ctrl-C` to create a stacktrace. + Development Environment ======================= diff --git a/VERSION b/VERSION index e815b861f..4312e0d0c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.15.1 +0.15.2 diff --git a/build.go b/build.go index dddc3b964..b3b7f5eee 100644 --- a/build.go +++ b/build.go @@ -380,6 +380,12 @@ func main() { } } + solarisMinVersion := GoVersion{Major: 1, Minor: 20, Patch: 0} + if env["GOARCH"] == "solaris" && !goVersion.AtLeast(solarisMinVersion) { + fmt.Fprintf(os.Stderr, "Detected version %s is too old, restic requires at least %s for Solaris\n", goVersion, solarisMinVersion) + os.Exit(1) + } + verbosePrintf("detected Go version %v\n", goVersion) preserveSymbols := false diff --git a/changelog/0.15.2_2023-04-24/issue-2260 b/changelog/0.15.2_2023-04-24/issue-2260 new file mode 100644 index 000000000..da4fe8e99 --- /dev/null +++ b/changelog/0.15.2_2023-04-24/issue-2260 @@ -0,0 +1,12 @@ +Bugfix: Sanitize filenames printed by `backup` during processing + +The `backup` command would previously not sanitize the filenames it printed +during processing, potentially causing newlines or terminal control characters +to mangle the status output or even change the state of a terminal. + +Filenames are now checked and quoted if they contain non-printable or +non-Unicode characters. + +https://github.com/restic/restic/issues/2260 +https://github.com/restic/restic/issues/4191 +https://github.com/restic/restic/pull/4192 diff --git a/changelog/0.15.2_2023-04-24/issue-4211 b/changelog/0.15.2_2023-04-24/issue-4211 new file mode 100644 index 000000000..0d499977c --- /dev/null +++ b/changelog/0.15.2_2023-04-24/issue-4211 @@ -0,0 +1,8 @@ +Bugfix: Make `dump` interpret `--host` and `--path` correctly + +A regression in restic 0.15.0 caused `dump` to confuse its `--host=` and +`--path=` options: it looked for snapshots with paths called `` +from hosts called ``. It now treats the options as intended. + +https://github.com/restic/restic/issues/4211 +https://github.com/restic/restic/pull/4212 diff --git a/changelog/0.15.2_2023-04-24/issue-4239 b/changelog/0.15.2_2023-04-24/issue-4239 new file mode 100644 index 000000000..43d099e24 --- /dev/null +++ b/changelog/0.15.2_2023-04-24/issue-4239 @@ -0,0 +1,11 @@ +Bugfix: Correct number of blocks reported in mount point + +Restic mount points reported an incorrect number of 512-byte (POSIX standard) +blocks for files and links due to a rounding bug. In particular, empty files +were reported as taking one block instead of zero. + +The rounding is now fixed: the number of blocks reported is the file size +(or link target size) divided by 512 and rounded up to a whole number. + +https://github.com/restic/restic/issues/4239 +https://github.com/restic/restic/pull/4240 diff --git a/changelog/0.15.2_2023-04-24/issue-4253 b/changelog/0.15.2_2023-04-24/issue-4253 new file mode 100644 index 000000000..d9109f988 --- /dev/null +++ b/changelog/0.15.2_2023-04-24/issue-4253 @@ -0,0 +1,18 @@ +Bugfix: Minimize risk of spurious filesystem loops with `mount` + +When a backup contains a directory that has the same name as its parent, say +`a/b/b`, and the GNU `find` command was run on this backup in a restic mount, +`find` would refuse to traverse the lowest `b` directory, instead printing +`File system loop detected`. This was due to the way the restic mount command +generates inode numbers for directories in the mount point. + +The rule for generating these inode numbers was changed in 0.15.0. It has +now been changed again to avoid this issue. A perfect rule does not exist, +but the probability of this behavior occurring is now extremely small. + +When it does occur, the mount point is not broken, and scripts that traverse +the mount point should work as long as they don't rely on inode numbers for +detecting filesystem loops. + +https://github.com/restic/restic/issues/4253 +https://github.com/restic/restic/pull/4255 diff --git a/changelog/0.15.2_2023-04-24/issue-4275 b/changelog/0.15.2_2023-04-24/issue-4275 new file mode 100644 index 000000000..944797b85 --- /dev/null +++ b/changelog/0.15.2_2023-04-24/issue-4275 @@ -0,0 +1,4 @@ +Security: Update golang.org/x/net to address CVE-2022-41723 + +https://github.com/restic/restic/issues/4275 +https://github.com/restic/restic/pull/4213 diff --git a/changelog/unreleased/pull-4180 b/changelog/0.15.2_2023-04-24/pull-4180 similarity index 55% rename from changelog/unreleased/pull-4180 rename to changelog/0.15.2_2023-04-24/pull-4180 index ff43feb2b..511974963 100644 --- a/changelog/unreleased/pull-4180 +++ b/changelog/0.15.2_2023-04-24/pull-4180 @@ -1,5 +1,6 @@ Enhancement: Add release binaries for riscv64 architecture on Linux -We've added release binaries for riscv64 architecture on Linux. +Builds for the `riscv64` architecture on Linux are now included in the +release binaries. https://github.com/restic/restic/pull/4180 diff --git a/changelog/0.15.2_2023-04-24/pull-4219 b/changelog/0.15.2_2023-04-24/pull-4219 new file mode 100644 index 000000000..25da7058b --- /dev/null +++ b/changelog/0.15.2_2023-04-24/pull-4219 @@ -0,0 +1,5 @@ +Enhancement: Upgrade Minio to version 7.0.49 + +The upgraded version now allows use of the `ap-southeast-4` region (Melbourne). + +https://github.com/restic/restic/pull/4219 diff --git a/changelog/unreleased/issue-1759 b/changelog/unreleased/issue-1759 new file mode 100644 index 000000000..1b698f845 --- /dev/null +++ b/changelog/unreleased/issue-1759 @@ -0,0 +1,20 @@ +Enhancement: Add `repair index` and `repair snapshots` commands + +The `rebuild-index` command has been renamed to `repair index`. The old name +will still work, but is deprecated. + +When a snapshot was damaged, the only option up to now was to completely forget +the snapshot, even if only some unimportant file was damaged. + +We've added a `repair snapshots` command, which can repair snapshots by removing +damaged directories and missing files contents. Note that using this command +can lead to data loss! Please see the "Troubleshooting" section in the documentation +for more details. + +https://github.com/restic/restic/issues/1759 +https://github.com/restic/restic/issues/1714 +https://github.com/restic/restic/issues/1798 +https://github.com/restic/restic/issues/2334 +https://github.com/restic/restic/pull/2876 +https://forum.restic.net/t/corrupted-repo-how-to-repair/799 +https://forum.restic.net/t/recovery-options-for-damaged-repositories/1571 diff --git a/changelog/unreleased/issue-2565 b/changelog/unreleased/issue-2565 new file mode 100644 index 000000000..4150dcda4 --- /dev/null +++ b/changelog/unreleased/issue-2565 @@ -0,0 +1,8 @@ +Bugfix: Restic forget --keep-* options now interpret "-1" as "forever" + +Restic would forget snapshots that should have been kept when "-1" was +used as a value for --keep-* options. It now interprets "-1" as forever, +e.g. an option like --keep-monthly -1 will keep all monthly snapshots. + +https://github.com/restic/restic/issues/2565 +https://github.com/restic/restic/pull/4234 diff --git a/changelog/unreleased/issue-3627 b/changelog/unreleased/issue-3627 new file mode 100644 index 000000000..edbbdbb33 --- /dev/null +++ b/changelog/unreleased/issue-3627 @@ -0,0 +1,9 @@ +Enhancement: Show progress bar during restore + +The `restore` command now shows a progress report while restoring files. + +Example: [0:42] 5.76% 23 files 12.98 MiB, total 3456 files 23.54 GiB + +https://github.com/restic/restic/issues/3627 +https://github.com/restic/restic/pull/3991 +https://forum.restic.net/t/progress-bar-for-restore/5210 diff --git a/changelog/unreleased/issue-4211 b/changelog/unreleased/issue-4211 deleted file mode 100644 index 45b7aee83..000000000 --- a/changelog/unreleased/issue-4211 +++ /dev/null @@ -1,8 +0,0 @@ -Bugfix: Restic dump now interprets --host and --path correctly - -Restic dump previously confused its --host= and --path= -options: it looked for snapshots with paths called from hosts -called . It now treats the options as intended. - -https://github.com/restic/restic/issues/4211 -https://github.com/restic/restic/pull/4212 diff --git a/changelog/unreleased/issue-719 b/changelog/unreleased/issue-719 new file mode 100644 index 000000000..4f28ea83c --- /dev/null +++ b/changelog/unreleased/issue-719 @@ -0,0 +1,8 @@ +Enhancement: Add --retry-lock option + +This option allows to specify a duration for which restic will wait if there +already exists a conflicting lock within the repository. + +https://github.com/restic/restic/issues/719 +https://github.com/restic/restic/pull/2214 +https://github.com/restic/restic/pull/4107 diff --git a/changelog/unreleased/pull-4166 b/changelog/unreleased/pull-4166 new file mode 100644 index 000000000..6714fdf7f --- /dev/null +++ b/changelog/unreleased/pull-4166 @@ -0,0 +1,7 @@ +Enhancement: Cancel current command if cache becomes unusable + +If the cache directory was removed or ran out of space while restic was +running, this caused further caching attempts to fail and drastically slow down +the command execution. Now, the currently running command is canceled instead. + +https://github.com/restic/restic/pull/4166 diff --git a/changelog/unreleased/pull-4201 b/changelog/unreleased/pull-4201 new file mode 100644 index 000000000..500bbdbb1 --- /dev/null +++ b/changelog/unreleased/pull-4201 @@ -0,0 +1,9 @@ +Change: Require Go 1.20 for Solaris builds + +Building restic on Solaris now requires Go 1.20, as the library used to access +Azure uses the mmap syscall, which is only available on Solaris starting from +Go 1.20. + +All other platforms continue to build with Go 1.18. + +https://github.com/restic/restic/pull/4201 diff --git a/changelog/unreleased/pull-4219 b/changelog/unreleased/pull-4219 deleted file mode 100644 index 7d20c3607..000000000 --- a/changelog/unreleased/pull-4219 +++ /dev/null @@ -1,5 +0,0 @@ -Enhancement: Upgrade Minio to 7.0.49 - -Upgraded to allow use of the ap-southeast-4 region (Melbourne) - -https://github.com/restic/restic/pull/4219 diff --git a/changelog/unreleased/pull-4220 b/changelog/unreleased/pull-4220 new file mode 100644 index 000000000..787b6ba2d --- /dev/null +++ b/changelog/unreleased/pull-4220 @@ -0,0 +1,5 @@ +Enhancement: Add jq to container image + +The Docker container image now contains jq which can be useful when restic outputs json data. + +https://github.com/restic/restic/pull/4220 diff --git a/changelog/unreleased/pull-4304 b/changelog/unreleased/pull-4304 new file mode 100644 index 000000000..ca3c7a8db --- /dev/null +++ b/changelog/unreleased/pull-4304 @@ -0,0 +1,5 @@ +Bugfix: Avoid lock refresh issues with slow network connections + +On network connections with a low upload speed, restic could often fail backups and other operations with `Fatal: failed to refresh lock in time`. We've reworked the lock refresh to avoid this error. + +https://github.com/restic/restic/pull/4304 diff --git a/changelog/unreleased/pull-4318 b/changelog/unreleased/pull-4318 new file mode 100644 index 000000000..198b972d3 --- /dev/null +++ b/changelog/unreleased/pull-4318 @@ -0,0 +1,8 @@ +Bugfix: Correctly clean up status bar output of the `backup` command + +Due to a regression in restic 0.15.2, the status bar of the `backup` command +could leave some output behind. This happened if filenames were printed that +are wider than the current terminal width. This has been fixed. + +https://github.com/restic/restic/issues/4319 +https://github.com/restic/restic/pull/4318 diff --git a/cmd/restic/cleanup.go b/cmd/restic/cleanup.go index 61af72802..75933fe96 100644 --- a/cmd/restic/cleanup.go +++ b/cmd/restic/cleanup.go @@ -62,6 +62,12 @@ func CleanupHandler(c <-chan os.Signal) { debug.Log("signal %v received, cleaning up", s) Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s) + if val, _ := os.LookupEnv("RESTIC_DEBUG_STACKTRACE_SIGINT"); val != "" { + _, _ = os.Stderr.WriteString("\n--- STACKTRACE START ---\n\n") + _, _ = os.Stderr.WriteString(debug.DumpStacktrace()) + _, _ = os.Stderr.WriteString("\n--- STACKTRACE END ---\n") + } + code := 0 if s == syscall.SIGINT { @@ -78,5 +84,6 @@ func CleanupHandler(c <-chan os.Signal) { // given exit code. func Exit(code int) { code = RunCleanupHandlers(code) + debug.Log("exiting with status code %d", code) os.Exit(code) } diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 1244e2ed1..fcaff304d 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -506,7 +506,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter if !gopts.JSON { progressPrinter.V("lock repository") } - lock, ctx, err := lockRepo(ctx, repo) + lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index e7253e5b6..771731a58 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -45,7 +45,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { if !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index d56f7d0c9..b9f3199b2 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -211,7 +211,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args if !gopts.NoLock { Verbosef("create exclusive lock for repository\n") var lock *restic.Lock - lock, ctx, err = lockRepoExclusive(ctx, repo) + lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err @@ -245,7 +245,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args } if suggestIndexRebuild { - Printf("Duplicate packs/old indexes are non-critical, you can run `restic rebuild-index' to correct this.\n") + Printf("Duplicate packs/old indexes are non-critical, you can run `restic repair index' to correct this.\n") } if mixedFound { Printf("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n") diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 2f095972a..13767d98a 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -74,14 +74,14 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] if !gopts.NoLock { var srcLock *restic.Lock - srcLock, ctx, err = lockRepo(ctx, srcRepo) + srcLock, ctx, err = lockRepo(ctx, srcRepo, gopts.RetryLock, gopts.JSON) defer unlockRepo(srcLock) if err != nil { return err } } - dstLock, ctx, err := lockRepo(ctx, dstRepo) + dstLock, ctx, err := lockRepo(ctx, dstRepo, gopts.RetryLock, gopts.JSON) defer unlockRepo(dstLock) if err != nil { return err diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index c8626d46c..deade6d22 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -156,7 +156,7 @@ func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error if !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err @@ -462,7 +462,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er if !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index 0000fd18a..0861a7103 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -334,7 +334,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args [] if !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index cda7b65b9..34313f582 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -132,7 +132,7 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] if !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index e5457c3be..3ef5f26bf 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -575,7 +575,7 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args [] if !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index fbe4c1c8a..eb8e7adde 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -67,12 +67,12 @@ func init() { cmdRoot.AddCommand(cmdForget) f := cmdForget.Flags() - f.IntVarP(&forgetOptions.Last, "keep-last", "l", 0, "keep the last `n` snapshots") - f.IntVarP(&forgetOptions.Hourly, "keep-hourly", "H", 0, "keep the last `n` hourly snapshots") - f.IntVarP(&forgetOptions.Daily, "keep-daily", "d", 0, "keep the last `n` daily snapshots") - f.IntVarP(&forgetOptions.Weekly, "keep-weekly", "w", 0, "keep the last `n` weekly snapshots") - f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots") - f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots") + f.IntVarP(&forgetOptions.Last, "keep-last", "l", 0, "keep the last `n` snapshots (use '-1' to keep all snapshots)") + f.IntVarP(&forgetOptions.Hourly, "keep-hourly", "H", 0, "keep the last `n` hourly snapshots (use '-1' to keep all hourly snapshots)") + f.IntVarP(&forgetOptions.Daily, "keep-daily", "d", 0, "keep the last `n` daily snapshots (use '-1' to keep all daily snapshots)") + f.IntVarP(&forgetOptions.Weekly, "keep-weekly", "w", 0, "keep the last `n` weekly snapshots (use '-1' to keep all weekly snapshots)") + f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots (use '-1' to keep all monthly snapshots)") + f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots (use '-1' to keep all yearly snapshots)") f.VarP(&forgetOptions.Within, "keep-within", "", "keep snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot") f.VarP(&forgetOptions.WithinHourly, "keep-within-hourly", "", "keep hourly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot") f.VarP(&forgetOptions.WithinDaily, "keep-within-daily", "", "keep daily snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot") @@ -99,8 +99,29 @@ func init() { addPruneOptions(cmdForget) } +func verifyForgetOptions(opts *ForgetOptions) error { + if opts.Last < -1 || opts.Hourly < -1 || opts.Daily < -1 || opts.Weekly < -1 || + opts.Monthly < -1 || opts.Yearly < -1 { + return errors.Fatal("negative values other than -1 are not allowed for --keep-*") + } + + for _, d := range []restic.Duration{opts.Within, opts.WithinHourly, opts.WithinDaily, + opts.WithinMonthly, opts.WithinWeekly, opts.WithinYearly} { + if d.Hours < 0 || d.Days < 0 || d.Months < 0 || d.Years < 0 { + return errors.Fatal("durations containing negative values are not allowed for --keep-within*") + } + } + + return nil +} + func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, args []string) error { - err := verifyPruneOptions(&pruneOptions) + err := verifyForgetOptions(&opts) + if err != nil { + return err + } + + err = verifyPruneOptions(&pruneOptions) if err != nil { return err } @@ -116,7 +137,7 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg if !opts.DryRun || !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepoExclusive(ctx, repo) + lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_forget_test.go b/cmd/restic/cmd_forget_test.go new file mode 100644 index 000000000..9fd5c7bb0 --- /dev/null +++ b/cmd/restic/cmd_forget_test.go @@ -0,0 +1,65 @@ +package main + +import ( + "testing" + + "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" +) + +func TestForgetOptionValues(t *testing.T) { + const negValErrorMsg = "Fatal: negative values other than -1 are not allowed for --keep-*" + const negDurationValErrorMsg = "Fatal: durations containing negative values are not allowed for --keep-within*" + testCases := []struct { + input ForgetOptions + expectsError bool + errorMsg string + }{ + {ForgetOptions{Last: 1}, false, ""}, + {ForgetOptions{Hourly: 1}, false, ""}, + {ForgetOptions{Daily: 1}, false, ""}, + {ForgetOptions{Weekly: 1}, false, ""}, + {ForgetOptions{Monthly: 1}, false, ""}, + {ForgetOptions{Yearly: 1}, false, ""}, + {ForgetOptions{Last: 0}, false, ""}, + {ForgetOptions{Hourly: 0}, false, ""}, + {ForgetOptions{Daily: 0}, false, ""}, + {ForgetOptions{Weekly: 0}, false, ""}, + {ForgetOptions{Monthly: 0}, false, ""}, + {ForgetOptions{Yearly: 0}, false, ""}, + {ForgetOptions{Last: -1}, false, ""}, + {ForgetOptions{Hourly: -1}, false, ""}, + {ForgetOptions{Daily: -1}, false, ""}, + {ForgetOptions{Weekly: -1}, false, ""}, + {ForgetOptions{Monthly: -1}, false, ""}, + {ForgetOptions{Yearly: -1}, false, ""}, + {ForgetOptions{Last: -2}, true, negValErrorMsg}, + {ForgetOptions{Hourly: -2}, true, negValErrorMsg}, + {ForgetOptions{Daily: -2}, true, negValErrorMsg}, + {ForgetOptions{Weekly: -2}, true, negValErrorMsg}, + {ForgetOptions{Monthly: -2}, true, negValErrorMsg}, + {ForgetOptions{Yearly: -2}, true, negValErrorMsg}, + {ForgetOptions{Within: restic.ParseDurationOrPanic("1y2m3d3h")}, false, ""}, + {ForgetOptions{WithinHourly: restic.ParseDurationOrPanic("1y2m3d3h")}, false, ""}, + {ForgetOptions{WithinDaily: restic.ParseDurationOrPanic("1y2m3d3h")}, false, ""}, + {ForgetOptions{WithinWeekly: restic.ParseDurationOrPanic("1y2m3d3h")}, false, ""}, + {ForgetOptions{WithinMonthly: restic.ParseDurationOrPanic("2y4m6d8h")}, false, ""}, + {ForgetOptions{WithinYearly: restic.ParseDurationOrPanic("2y4m6d8h")}, false, ""}, + {ForgetOptions{Within: restic.ParseDurationOrPanic("-1y2m3d3h")}, true, negDurationValErrorMsg}, + {ForgetOptions{WithinHourly: restic.ParseDurationOrPanic("1y-2m3d3h")}, true, negDurationValErrorMsg}, + {ForgetOptions{WithinDaily: restic.ParseDurationOrPanic("1y2m-3d3h")}, true, negDurationValErrorMsg}, + {ForgetOptions{WithinWeekly: restic.ParseDurationOrPanic("1y2m3d-3h")}, true, negDurationValErrorMsg}, + {ForgetOptions{WithinMonthly: restic.ParseDurationOrPanic("-2y4m6d8h")}, true, negDurationValErrorMsg}, + {ForgetOptions{WithinYearly: restic.ParseDurationOrPanic("2y-4m6d8h")}, true, negDurationValErrorMsg}, + } + + for _, testCase := range testCases { + err := verifyForgetOptions(&testCase.input) + if testCase.expectsError { + rtest.Assert(t, err != nil, "should have returned error for input %+v", testCase.input) + rtest.Equals(t, testCase.errorMsg, err.Error()) + } else { + rtest.Assert(t, err == nil, "expected no error for input %+v", testCase.input) + } + } +} diff --git a/cmd/restic/cmd_generate.go b/cmd/restic/cmd_generate.go index 959a9d518..2e944ad37 100644 --- a/cmd/restic/cmd_generate.go +++ b/cmd/restic/cmd_generate.go @@ -63,22 +63,30 @@ func writeManpages(dir string) error { } func writeBashCompletion(file string) error { - Verbosef("writing bash completion file to %v\n", file) + if stdoutIsTerminal() { + Verbosef("writing bash completion file to %v\n", file) + } return cmdRoot.GenBashCompletionFile(file) } func writeFishCompletion(file string) error { - Verbosef("writing fish completion file to %v\n", file) + if stdoutIsTerminal() { + Verbosef("writing fish completion file to %v\n", file) + } return cmdRoot.GenFishCompletionFile(file, true) } func writeZSHCompletion(file string) error { - Verbosef("writing zsh completion file to %v\n", file) + if stdoutIsTerminal() { + Verbosef("writing zsh completion file to %v\n", file) + } return cmdRoot.GenZshCompletionFile(file) } func writePowerShellCompletion(file string) error { - Verbosef("writing powershell completion file to %v\n", file) + if stdoutIsTerminal() { + Verbosef("writing powershell completion file to %v\n", file) + } return cmdRoot.GenPowerShellCompletionFile(file) } diff --git a/cmd/restic/cmd_init.go b/cmd/restic/cmd_init.go index 2932870e8..a878f3e16 100644 --- a/cmd/restic/cmd_init.go +++ b/cmd/restic/cmd_init.go @@ -102,7 +102,12 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args [] } if !gopts.JSON { - Verbosef("created restic repository %v at %s\n", s.Config().ID[:10], location.StripPassword(gopts.Repo)) + Verbosef("created restic repository %v at %s", s.Config().ID[:10], location.StripPassword(gopts.Repo)) + if opts.CopyChunkerParameters && chunkerPolynomial != nil { + Verbosef(" with chunker parameters copied from secondary repository\n") + } else { + Verbosef("\n") + } Verbosef("\n") Verbosef("Please note that knowledge of your password is required to access\n") Verbosef("the repository. Losing your password means that your data is\n") diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index 88b6d5c0c..62521d762 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -212,7 +212,7 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error { switch args[0] { case "list": - lock, ctx, err := lockRepo(ctx, repo) + lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err @@ -220,7 +220,7 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error { return listKeys(ctx, repo, gopts) case "add": - lock, ctx, err := lockRepo(ctx, repo) + lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err @@ -228,7 +228,7 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error { return addKey(ctx, repo, gopts) case "remove": - lock, ctx, err := lockRepoExclusive(ctx, repo) + lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err @@ -241,7 +241,7 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error { return deleteKey(ctx, repo, id) case "passwd": - lock, ctx, err := lockRepoExclusive(ctx, repo) + lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go index 4809092c0..bd02cedc7 100644 --- a/cmd/restic/cmd_list.go +++ b/cmd/restic/cmd_list.go @@ -31,19 +31,19 @@ func init() { cmdRoot.AddCommand(cmdList) } -func runList(ctx context.Context, cmd *cobra.Command, opts GlobalOptions, args []string) error { +func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args []string) error { if len(args) != 1 { return errors.Fatal("type not specified, usage: " + cmd.Use) } - repo, err := OpenRepository(ctx, opts) + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } - if !opts.NoLock && args[0] != "locks" { + if !gopts.NoLock && args[0] != "locks" { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_migrate.go b/cmd/restic/cmd_migrate.go index 6d614be39..fd2e762c0 100644 --- a/cmd/restic/cmd_migrate.go +++ b/cmd/restic/cmd_migrate.go @@ -122,7 +122,7 @@ func runMigrate(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, a return err } - lock, ctx, err := lockRepoExclusive(ctx, repo) + lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index 0501bfe89..ec3662d5c 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -123,7 +123,7 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args if !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index f59be2967..1138bb55b 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -167,7 +167,7 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error opts.unsafeRecovery = true } - lock, ctx, err := lockRepoExclusive(ctx, repo) + lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err @@ -488,7 +488,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Reposi // Pack size does not fit and pack is needed => error // If the pack is not needed, this is no error, the pack can // and will be simply removed, see below. - Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic rebuild-index'.\n", + Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic repair index'.\n", id.Str(), p.unusedSize+p.usedSize, packSize) return errorSizeNotMatching } diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index 65f4c8750..85dcc23d7 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -46,7 +46,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error { return err } - lock, ctx, err := lockRepo(ctx, repo) + lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_repair.go b/cmd/restic/cmd_repair.go new file mode 100644 index 000000000..aefe02f3c --- /dev/null +++ b/cmd/restic/cmd_repair.go @@ -0,0 +1,14 @@ +package main + +import ( + "github.com/spf13/cobra" +) + +var cmdRepair = &cobra.Command{ + Use: "repair", + Short: "Repair the repository", +} + +func init() { + cmdRoot.AddCommand(cmdRepair) +} diff --git a/cmd/restic/cmd_rebuild_index.go b/cmd/restic/cmd_repair_index.go similarity index 73% rename from cmd/restic/cmd_rebuild_index.go rename to cmd/restic/cmd_repair_index.go index 6d49cb917..25d6b1cab 100644 --- a/cmd/restic/cmd_rebuild_index.go +++ b/cmd/restic/cmd_repair_index.go @@ -7,15 +7,15 @@ import ( "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" - "github.com/spf13/cobra" + "github.com/spf13/pflag" ) -var cmdRebuildIndex = &cobra.Command{ - Use: "rebuild-index [flags]", +var cmdRepairIndex = &cobra.Command{ + Use: "index [flags]", Short: "Build a new index", Long: ` -The "rebuild-index" command creates a new index based on the pack files in the +The "repair index" command creates a new index based on the pack files in the repository. EXIT STATUS @@ -25,31 +25,43 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runRebuildIndex(cmd.Context(), rebuildIndexOptions, globalOptions) + return runRebuildIndex(cmd.Context(), repairIndexOptions, globalOptions) }, } -// RebuildIndexOptions collects all options for the rebuild-index command. -type RebuildIndexOptions struct { +var cmdRebuildIndex = &cobra.Command{ + Use: "rebuild-index [flags]", + Short: cmdRepairIndex.Short, + Long: cmdRepairIndex.Long, + Deprecated: `Use "repair index" instead`, + DisableAutoGenTag: true, + RunE: cmdRepairIndex.RunE, +} + +// RepairIndexOptions collects all options for the repair index command. +type RepairIndexOptions struct { ReadAllPacks bool } -var rebuildIndexOptions RebuildIndexOptions +var repairIndexOptions RepairIndexOptions func init() { + cmdRepair.AddCommand(cmdRepairIndex) + // add alias for old name cmdRoot.AddCommand(cmdRebuildIndex) - f := cmdRebuildIndex.Flags() - f.BoolVar(&rebuildIndexOptions.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch") + for _, f := range []*pflag.FlagSet{cmdRepairIndex.Flags(), cmdRebuildIndex.Flags()} { + f.BoolVar(&repairIndexOptions.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch") + } } -func runRebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions) error { +func runRebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions) error { repo, err := OpenRepository(ctx, gopts) if err != nil { return err } - lock, ctx, err := lockRepoExclusive(ctx, repo) + lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err @@ -58,7 +70,7 @@ func runRebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts Global return rebuildIndex(ctx, opts, gopts, repo, restic.NewIDSet()) } -func rebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error { +func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error { var obsoleteIndexes restic.IDs packSizeFromList := make(map[restic.ID]int64) packSizeFromIndex := make(map[restic.ID]int64) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go new file mode 100644 index 000000000..5e9ec4130 --- /dev/null +++ b/cmd/restic/cmd_repair_snapshots.go @@ -0,0 +1,176 @@ +package main + +import ( + "context" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/walker" + + "github.com/spf13/cobra" +) + +var cmdRepairSnapshots = &cobra.Command{ + Use: "snapshots [flags] [snapshot ID] [...]", + Short: "Repair snapshots", + Long: ` +The "repair snapshots" command repairs broken snapshots. It scans the given +snapshots and generates new ones with damaged directories and file contents +removed. If the broken snapshots are deleted, a prune run will be able to +clean up the repository. + +The command depends on a correct index, thus make sure to run "repair index" +first! + + +WARNING +======= + +Repairing and deleting broken snapshots causes data loss! It will remove broken +directories and modify broken files in the modified snapshots. + +If the contents of directories and files are still available, the better option +is to run "backup" which in that case is able to heal existing snapshots. Only +use the "repair snapshots" command if you need to recover an old and broken +snapshot! + +EXIT STATUS +=========== + +Exit status is 0 if the command was successful, and non-zero if there was any error. +`, + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + return runRepairSnapshots(cmd.Context(), globalOptions, repairSnapshotOptions, args) + }, +} + +// RepairOptions collects all options for the repair command. +type RepairOptions struct { + DryRun bool + Forget bool + + restic.SnapshotFilter +} + +var repairSnapshotOptions RepairOptions + +func init() { + cmdRepair.AddCommand(cmdRepairSnapshots) + flags := cmdRepairSnapshots.Flags() + + flags.BoolVarP(&repairSnapshotOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done") + flags.BoolVarP(&repairSnapshotOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones") + + initMultiSnapshotFilter(flags, &repairSnapshotOptions.SnapshotFilter, true) +} + +func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOptions, args []string) error { + repo, err := OpenRepository(ctx, globalOptions) + if err != nil { + return err + } + + if !opts.DryRun { + var lock *restic.Lock + var err error + lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) + defer unlockRepo(lock) + if err != nil { + return err + } + } else { + repo.SetDryRun() + } + + snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) + if err != nil { + return err + } + + if err := repo.LoadIndex(ctx); err != nil { + return err + } + + // Three error cases are checked: + // - tree is a nil tree (-> will be replaced by an empty tree) + // - trees which cannot be loaded (-> the tree contents will be removed) + // - files whose contents are not fully available (-> file will be modified) + rewriter := walker.NewTreeRewriter(walker.RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { + if node.Type != "file" { + return node + } + + ok := true + var newContent restic.IDs = restic.IDs{} + var newSize uint64 + // check all contents and remove if not available + for _, id := range node.Content { + if size, found := repo.LookupBlobSize(id, restic.DataBlob); !found { + ok = false + } else { + newContent = append(newContent, id) + newSize += uint64(size) + } + } + if !ok { + Verbosef(" file %q: removed missing content\n", path) + } else if newSize != node.Size { + Verbosef(" file %q: fixed incorrect size\n", path) + } + // no-ops if already correct + node.Content = newContent + node.Size = newSize + return node + }, + RewriteFailedTree: func(nodeID restic.ID, path string, _ error) (restic.ID, error) { + if path == "/" { + Verbosef(" dir %q: not readable\n", path) + // remove snapshots with invalid root node + return restic.ID{}, nil + } + // If a subtree fails to load, remove it + Verbosef(" dir %q: replaced with empty directory\n", path) + emptyID, err := restic.SaveTree(ctx, repo, &restic.Tree{}) + if err != nil { + return restic.ID{}, err + } + return emptyID, nil + }, + AllowUnstableSerialization: true, + }) + + changedCount := 0 + for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) { + Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) + changed, err := filterAndReplaceSnapshot(ctx, repo, sn, + func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { + return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) + }, opts.DryRun, opts.Forget, "repaired") + if err != nil { + return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err) + } + if changed { + changedCount++ + } + } + + Verbosef("\n") + if changedCount == 0 { + if !opts.DryRun { + Verbosef("no snapshots were modified\n") + } else { + Verbosef("no snapshots would be modified\n") + } + } else { + if !opts.DryRun { + Verbosef("modified %v snapshots\n", changedCount) + } else { + Verbosef("would modify %v snapshots\n", changedCount) + } + } + + return nil +} diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 579711662..a3e602c8f 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -3,6 +3,7 @@ package main import ( "context" "strings" + "sync" "time" "github.com/restic/restic/internal/debug" @@ -10,6 +11,9 @@ import ( "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restorer" + "github.com/restic/restic/internal/ui" + restoreui "github.com/restic/restic/internal/ui/restore" + "github.com/restic/restic/internal/ui/termstatus" "github.com/spf13/cobra" ) @@ -31,7 +35,31 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runRestore(cmd.Context(), restoreOptions, globalOptions, args) + ctx := cmd.Context() + var wg sync.WaitGroup + cancelCtx, cancel := context.WithCancel(ctx) + defer func() { + // shutdown termstatus + cancel() + wg.Wait() + }() + + term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet) + wg.Add(1) + go func() { + defer wg.Done() + term.Run(cancelCtx) + }() + + // allow usage of warnf / verbosef + prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr + defer func() { + globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr + }() + stdioWrapper := ui.NewStdioWrapper(term) + globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr() + + return runRestore(ctx, restoreOptions, globalOptions, term, args) }, } @@ -64,7 +92,9 @@ func init() { flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content") } -func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, args []string) error { +func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, + term *termstatus.Terminal, args []string) error { + hasExcludes := len(opts.Exclude) > 0 || len(opts.InsensitiveExclude) > 0 hasIncludes := len(opts.Include) > 0 || len(opts.InsensitiveInclude) > 0 @@ -124,7 +154,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, a if !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err @@ -145,7 +175,12 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, a return err } - res := restorer.NewRestorer(ctx, repo, sn, opts.Sparse) + var progress *restoreui.Progress + if !globalOptions.Quiet && !globalOptions.JSON { + progress = restoreui.NewProgress(restoreui.NewProgressPrinter(term), calculateProgressInterval(!gopts.Quiet, gopts.JSON)) + } + + res := restorer.NewRestorer(ctx, repo, sn, opts.Sparse, progress) totalErrors := 0 res.Error = func(location string, err error) error { @@ -209,6 +244,10 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, a return err } + if progress != nil { + progress.Finish() + } + if totalErrors > 0 { return errors.Fatalf("There were %d errors\n", totalErrors) } diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index 0d9aa1c8c..c08797c48 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -87,36 +87,67 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti return true } + rewriter := walker.NewTreeRewriter(walker.RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { + if selectByName(path) { + return node + } + Verbosef(fmt.Sprintf("excluding %s\n", path)) + return nil + }, + DisableNodeCache: true, + }) + + return filterAndReplaceSnapshot(ctx, repo, sn, + func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { + return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) + }, opts.DryRun, opts.Forget, "rewrite") +} + +func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, filter func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error), dryRun bool, forget bool, addTag string) (bool, error) { + wg, wgCtx := errgroup.WithContext(ctx) repo.StartPackUploader(wgCtx, wg) var filteredTree restic.ID wg.Go(func() error { - filteredTree, err = walker.FilterTree(wgCtx, repo, "/", *sn.Tree, &walker.TreeFilterVisitor{ - SelectByName: selectByName, - PrintExclude: func(path string) { Verbosef(fmt.Sprintf("excluding %s\n", path)) }, - }) + var err error + filteredTree, err = filter(ctx, sn) if err != nil { return err } return repo.Flush(wgCtx) }) - err = wg.Wait() + err := wg.Wait() if err != nil { return false, err } + if filteredTree.IsNull() { + if dryRun { + Verbosef("would delete empty snapshot\n") + } else { + h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} + if err = repo.Backend().Remove(ctx, h); err != nil { + return false, err + } + debug.Log("removed empty snapshot %v", sn.ID()) + Verbosef("removed empty snapshot %v\n", sn.ID().Str()) + } + return true, nil + } + if filteredTree == *sn.Tree { debug.Log("Snapshot %v not modified", sn) return false, nil } debug.Log("Snapshot %v modified", sn) - if opts.DryRun { + if dryRun { Verbosef("would save new snapshot\n") - if opts.Forget { + if forget { Verbosef("would remove old snapshot\n") } @@ -125,10 +156,10 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti // Always set the original snapshot id as this essentially a new snapshot. sn.Original = sn.ID() - *sn.Tree = filteredTree + sn.Tree = &filteredTree - if !opts.Forget { - sn.AddTags([]string{"rewrite"}) + if !forget { + sn.AddTags([]string{addTag}) } // Save the new snapshot. @@ -136,8 +167,9 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti if err != nil { return false, err } + Verbosef("saved new snapshot %v\n", id.Str()) - if opts.Forget { + if forget { h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} if err = repo.Backend().Remove(ctx, h); err != nil { return false, err @@ -145,7 +177,6 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti debug.Log("removed old snapshot %v", sn.ID()) Verbosef("removed old snapshot %v\n", sn.ID().Str()) } - Verbosef("saved new snapshot %v\n", id.Str()) return true, nil } @@ -164,9 +195,9 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a var err error if opts.Forget { Verbosef("create exclusive lock for repository\n") - lock, ctx, err = lockRepoExclusive(ctx, repo) + lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) } else { - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) } defer unlockRepo(lock) if err != nil { diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index 2de8801cb..ba3644ee7 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -65,7 +65,7 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions if !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 55ba6f254..e8558d290 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -83,7 +83,7 @@ func runStats(ctx context.Context, gopts GlobalOptions, args []string) error { if !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_tag.go b/cmd/restic/cmd_tag.go index e5948ea02..fe4638547 100644 --- a/cmd/restic/cmd_tag.go +++ b/cmd/restic/cmd_tag.go @@ -111,7 +111,7 @@ func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []st if !gopts.NoLock { Verbosef("create exclusive lock for repository\n") var lock *restic.Lock - lock, ctx, err = lockRepoExclusive(ctx, repo) + lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 0d6079b04..a4e6c06f2 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -20,10 +20,12 @@ import ( "github.com/restic/restic/internal/backend/limiter" "github.com/restic/restic/internal/backend/local" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/logger" "github.com/restic/restic/internal/backend/rclone" "github.com/restic/restic/internal/backend/rest" "github.com/restic/restic/internal/backend/retry" "github.com/restic/restic/internal/backend/s3" + "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/backend/sftp" "github.com/restic/restic/internal/backend/smb" "github.com/restic/restic/internal/backend/swift" @@ -43,7 +45,7 @@ import ( "golang.org/x/term" ) -var version = "0.15.1-dev (compiled manually)" +var version = "0.15.2-dev (compiled manually)" // TimeFormat is the format used for all timestamps printed by restic. const TimeFormat = "2006-01-02 15:04:05" @@ -60,6 +62,7 @@ type GlobalOptions struct { Quiet bool Verbose int NoLock bool + RetryLock time.Duration JSON bool CacheDir string NoCache bool @@ -116,6 +119,7 @@ func init() { // use empty paremeter name as `-v, --verbose n` instead of the correct `--verbose=n` is confusing f.CountVarP(&globalOptions.Verbose, "verbose", "v", "be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2)") f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repository, this allows some operations on read-only repositories") + f.DurationVar(&globalOptions.RetryLock, "retry-lock", 0, "retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries)") f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it") f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache `directory`. (default: use system default cache directory)") f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache") @@ -123,7 +127,7 @@ func init() { f.StringVar(&globalOptions.TLSClientCertKeyFilename, "tls-client-cert", "", "path to a `file` containing PEM encoded TLS client certificate and private key") f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)") f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") - f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max)") + f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)") f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)") f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)") f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)") @@ -281,6 +285,7 @@ func Warnf(format string, args ...interface{}) { if err != nil { fmt.Fprintf(os.Stderr, "unable to write to stderr: %v\n", err) } + debug.Log(format, args...) } // resolvePassword determines the password to be used for opening the repository. @@ -767,6 +772,9 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio return nil, errors.Fatalf("unable to open repository at %v: %v", location.StripPassword(s), err) } + // wrap with debug logging and connection limiting + be = logger.New(sema.NewBackend(be)) + // wrap backend if a test specified an inner hook if gopts.backendInnerTestHook != nil { be, err = gopts.backendInnerTestHook(be) @@ -811,29 +819,36 @@ func create(ctx context.Context, s string, opts options.Options) (restic.Backend return nil, err } + var be restic.Backend switch loc.Scheme { case "local": - return local.Create(ctx, cfg.(local.Config)) + be, err = local.Create(ctx, cfg.(local.Config)) case "sftp": - return sftp.Create(ctx, cfg.(sftp.Config)) + be, err = sftp.Create(ctx, cfg.(sftp.Config)) case "smb": - return smb.Create(ctx, cfg.(smb.Config)) + be, err = smb.Create(ctx, cfg.(smb.Config)) case "s3": - return s3.Create(ctx, cfg.(s3.Config), rt) + be, err = s3.Create(ctx, cfg.(s3.Config), rt) case "gs": - return gs.Create(cfg.(gs.Config), rt) + be, err = gs.Create(ctx, cfg.(gs.Config), rt) case "azure": - return azure.Create(ctx, cfg.(azure.Config), rt) + be, err = azure.Create(ctx, cfg.(azure.Config), rt) case "swift": - return swift.Open(ctx, cfg.(swift.Config), rt) + be, err = swift.Open(ctx, cfg.(swift.Config), rt) case "b2": - return b2.Create(ctx, cfg.(b2.Config), rt) + be, err = b2.Create(ctx, cfg.(b2.Config), rt) case "rest": - return rest.Create(ctx, cfg.(rest.Config), rt) + be, err = rest.Create(ctx, cfg.(rest.Config), rt) case "rclone": - return rclone.Create(ctx, cfg.(rclone.Config)) + be, err = rclone.Create(ctx, cfg.(rclone.Config)) + default: + debug.Log("invalid repository scheme: %v", s) + return nil, errors.Fatalf("invalid scheme %q", loc.Scheme) } - debug.Log("invalid repository scheme: %v", s) - return nil, errors.Fatalf("invalid scheme %q", loc.Scheme) + if err != nil { + return nil, err + } + + return logger.New(sema.NewBackend(be)), nil } diff --git a/cmd/restic/integration_fuse_test.go b/cmd/restic/integration_fuse_test.go index a99064b8f..b69886024 100644 --- a/cmd/restic/integration_fuse_test.go +++ b/cmd/restic/integration_fuse_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -159,6 +160,11 @@ func TestMount(t *testing.T) { t.Skip("Skipping fuse tests") } + debugEnabled := debug.TestLogToStderr(t) + if debugEnabled { + defer debug.TestDisableLog(t) + } + env, cleanup := withTestEnvironment(t) // must list snapshots more than once env.gopts.backendTestHook = nil diff --git a/cmd/restic/integration_repair_snapshots_test.go b/cmd/restic/integration_repair_snapshots_test.go new file mode 100644 index 000000000..04ef6ad1d --- /dev/null +++ b/cmd/restic/integration_repair_snapshots_test.go @@ -0,0 +1,135 @@ +package main + +import ( + "context" + "hash/fnv" + "io" + "math/rand" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" +) + +func testRunRepairSnapshot(t testing.TB, gopts GlobalOptions, forget bool) { + opts := RepairOptions{ + Forget: forget, + } + + rtest.OK(t, runRepairSnapshots(context.TODO(), gopts, opts, nil)) +} + +func createRandomFile(t testing.TB, env *testEnvironment, path string, size int) { + fn := filepath.Join(env.testdata, path) + rtest.OK(t, os.MkdirAll(filepath.Dir(fn), 0o755)) + + h := fnv.New64() + _, err := h.Write([]byte(path)) + rtest.OK(t, err) + r := rand.New(rand.NewSource(int64(h.Sum64()))) + + f, err := os.OpenFile(fn, os.O_CREATE|os.O_RDWR, 0o644) + rtest.OK(t, err) + _, err = io.Copy(f, io.LimitReader(r, int64(size))) + rtest.OK(t, err) + rtest.OK(t, f.Close()) +} + +func TestRepairSnapshotsWithLostData(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + + createRandomFile(t, env, "foo/bar/file", 512*1024) + testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts) + testListSnapshots(t, env.gopts, 1) + // damage repository + removePacksExcept(env.gopts, t, restic.NewIDSet(), false) + + createRandomFile(t, env, "foo/bar/file2", 256*1024) + testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts) + snapshotIDs := testListSnapshots(t, env.gopts, 2) + testRunCheckMustFail(t, env.gopts) + + // repair but keep broken snapshots + testRunRebuildIndex(t, env.gopts) + testRunRepairSnapshot(t, env.gopts, false) + testListSnapshots(t, env.gopts, 4) + testRunCheckMustFail(t, env.gopts) + + // repository must be ok after removing the broken snapshots + testRunForget(t, env.gopts, snapshotIDs[0].String(), snapshotIDs[1].String()) + testListSnapshots(t, env.gopts, 2) + _, err := testRunCheckOutput(env.gopts) + rtest.OK(t, err) +} + +func TestRepairSnapshotsWithLostTree(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + + createRandomFile(t, env, "foo/bar/file", 12345) + testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts) + oldSnapshot := testListSnapshots(t, env.gopts, 1) + oldPacks := testRunList(t, "packs", env.gopts) + + // keep foo/bar unchanged + createRandomFile(t, env, "foo/bar2", 1024) + testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts) + testListSnapshots(t, env.gopts, 2) + + // remove tree for foo/bar and the now completely broken first snapshot + removePacks(env.gopts, t, restic.NewIDSet(oldPacks...)) + testRunForget(t, env.gopts, oldSnapshot[0].String()) + testRunCheckMustFail(t, env.gopts) + + // repair + testRunRebuildIndex(t, env.gopts) + testRunRepairSnapshot(t, env.gopts, true) + testListSnapshots(t, env.gopts, 1) + _, err := testRunCheckOutput(env.gopts) + rtest.OK(t, err) +} + +func TestRepairSnapshotsWithLostRootTree(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + + createRandomFile(t, env, "foo/bar/file", 12345) + testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts) + testListSnapshots(t, env.gopts, 1) + oldPacks := testRunList(t, "packs", env.gopts) + + // remove all trees + removePacks(env.gopts, t, restic.NewIDSet(oldPacks...)) + testRunCheckMustFail(t, env.gopts) + + // repair + testRunRebuildIndex(t, env.gopts) + testRunRepairSnapshot(t, env.gopts, true) + testListSnapshots(t, env.gopts, 0) + _, err := testRunCheckOutput(env.gopts) + rtest.OK(t, err) +} + +func TestRepairSnapshotsIntact(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + testSetupBackupData(t, env) + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{}, env.gopts) + oldSnapshotIDs := testListSnapshots(t, env.gopts, 1) + + // use an exclude that will not exclude anything + testRunRepairSnapshot(t, env.gopts, false) + snapshotIDs := testListSnapshots(t, env.gopts, 1) + rtest.Assert(t, reflect.DeepEqual(oldSnapshotIDs, snapshotIDs), "unexpected snapshot id mismatch %v vs. %v", oldSnapshotIDs, snapshotIDs) + testRunCheck(t, env.gopts) +} diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index c87722f02..211089253 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -71,6 +71,7 @@ func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts defer cleanup() } + opts.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true} backupErr := runBackup(ctx, opts, gopts, term, target) cancel() @@ -99,6 +100,13 @@ func testRunList(t testing.TB, tpe string, opts GlobalOptions) restic.IDs { return parseIDsFromReader(t, buf) } +func testListSnapshots(t testing.TB, opts GlobalOptions, expected int) restic.IDs { + t.Helper() + snapshotIDs := testRunList(t, "snapshots", opts) + rtest.Assert(t, len(snapshotIDs) == expected, "expected %v snapshot, got %v", expected, snapshotIDs) + return snapshotIDs +} + func testRunRestore(t testing.TB, opts GlobalOptions, dir string, snapshotID restic.ID) { testRunRestoreExcludes(t, opts, dir, snapshotID, nil) } @@ -112,7 +120,7 @@ func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths [ }, } - rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{"latest"})) + rtest.OK(t, runRestore(context.TODO(), opts, gopts, nil, []string{"latest"})) } func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, excludes []string) { @@ -121,7 +129,7 @@ func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snaps Exclude: excludes, } - rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{snapshotID.String()})) + rtest.OK(t, runRestore(context.TODO(), opts, gopts, nil, []string{snapshotID.String()})) } func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includes []string) { @@ -130,11 +138,11 @@ func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snaps Include: includes, } - rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{snapshotID.String()})) + rtest.OK(t, runRestore(context.TODO(), opts, gopts, nil, []string{snapshotID.String()})) } func testRunRestoreAssumeFailure(t testing.TB, snapshotID string, opts RestoreOptions, gopts GlobalOptions) error { - err := runRestore(context.TODO(), opts, gopts, []string{snapshotID}) + err := runRestore(context.TODO(), opts, gopts, nil, []string{snapshotID}) return err } @@ -163,6 +171,11 @@ func testRunCheckOutput(gopts GlobalOptions) (string, error) { return buf.String(), err } +func testRunCheckMustFail(t testing.TB, gopts GlobalOptions) { + _, err := testRunCheckOutput(gopts) + rtest.Assert(t, err != nil, "expected non nil error after check of damaged repository") +} + func testRunDiffOutput(gopts GlobalOptions, firstSnapshotID string, secondSnapshotID string) (string, error) { buf := bytes.NewBuffer(nil) @@ -187,7 +200,7 @@ func testRunRebuildIndex(t testing.TB, gopts GlobalOptions) { globalOptions.stdout = os.Stdout }() - rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{}, gopts)) + rtest.OK(t, runRebuildIndex(context.TODO(), RepairIndexOptions{}, gopts)) } func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string { @@ -362,6 +375,55 @@ func testBackup(t *testing.T, useFsSnapshot bool) { testRunCheck(t, env.gopts) } +func TestBackupWithRelativePath(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{} + + // first backup + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts) + snapshotIDs := testRunList(t, "snapshots", env.gopts) + rtest.Assert(t, len(snapshotIDs) == 1, "expected one snapshot, got %v", snapshotIDs) + firstSnapshotID := snapshotIDs[0] + + // second backup, implicit incremental + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts) + + // that the correct parent snapshot was used + latestSn, _ := testRunSnapshots(t, env.gopts) + rtest.Assert(t, latestSn != nil, "missing latest snapshot") + rtest.Assert(t, latestSn.Parent != nil && latestSn.Parent.Equal(firstSnapshotID), "second snapshot selected unexpected parent %v instead of %v", latestSn.Parent, firstSnapshotID) +} + +func TestBackupParentSelection(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{} + + // first backup + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata/0/0"}, opts, env.gopts) + snapshotIDs := testRunList(t, "snapshots", env.gopts) + rtest.Assert(t, len(snapshotIDs) == 1, "expected one snapshot, got %v", snapshotIDs) + firstSnapshotID := snapshotIDs[0] + + // second backup, sibling path + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata/0/tests"}, opts, env.gopts) + snapshotIDs = testRunList(t, "snapshots", env.gopts) + rtest.Assert(t, len(snapshotIDs) == 2, "expected two snapshots, got %v", snapshotIDs) + + // third backup, incremental for the first backup + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata/0/0"}, opts, env.gopts) + + // test that the correct parent snapshot was used + latestSn, _ := testRunSnapshots(t, env.gopts) + rtest.Assert(t, latestSn != nil, "missing latest snapshot") + rtest.Assert(t, latestSn.Parent != nil && latestSn.Parent.Equal(firstSnapshotID), "third snapshot selected unexpected parent %v instead of %v", latestSn.Parent, firstSnapshotID) +} + func TestDryRunBackup(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() @@ -436,7 +498,16 @@ func TestBackupNonExistingFile(t *testing.T) { testRunBackup(t, "", dirs, opts, env.gopts) } -func removePacksExcept(gopts GlobalOptions, t *testing.T, keep restic.IDSet, removeTreePacks bool) { +func removePacks(gopts GlobalOptions, t testing.TB, remove restic.IDSet) { + r, err := OpenRepository(context.TODO(), gopts) + rtest.OK(t, err) + + for id := range remove { + rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()})) + } +} + +func removePacksExcept(gopts GlobalOptions, t testing.TB, keep restic.IDSet, removeTreePacks bool) { r, err := OpenRepository(context.TODO(), gopts) rtest.OK(t, err) @@ -1454,8 +1525,8 @@ func testRebuildIndex(t *testing.T, backendTestHook backendWrapper) { t.Fatalf("expected no error from checker for test repository, got %v", err) } - if !strings.Contains(out, "restic rebuild-index") { - t.Fatalf("did not find hint for rebuild-index command") + if !strings.Contains(out, "restic repair index") { + t.Fatalf("did not find hint for repair index command") } env.gopts.backendTestHook = backendTestHook @@ -1468,7 +1539,7 @@ func testRebuildIndex(t *testing.T, backendTestHook backendWrapper) { } if err != nil { - t.Fatalf("expected no error from checker after rebuild-index, got: %v", err) + t.Fatalf("expected no error from checker after repair index, got: %v", err) } } @@ -1549,7 +1620,7 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) { env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return &appendOnlyBackend{r}, nil } - err := runRebuildIndex(context.TODO(), RebuildIndexOptions{}, env.gopts) + err := runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts) if err == nil { t.Error("expected rebuildIndex to fail") } @@ -1837,8 +1908,8 @@ func TestListOnce(t *testing.T) { testRunPrune(t, env.gopts, pruneOpts) rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil)) - rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{}, env.gopts)) - rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{ReadAllPacks: true}, env.gopts)) + rtest.OK(t, runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts)) + rtest.OK(t, runRebuildIndex(context.TODO(), RepairIndexOptions{ReadAllPacks: true}, env.gopts)) } func TestHardLink(t *testing.T) { diff --git a/cmd/restic/lock.go b/cmd/restic/lock.go index f39a08db6..450922704 100644 --- a/cmd/restic/lock.go +++ b/cmd/restic/lock.go @@ -21,17 +21,29 @@ var globalLocks struct { sync.Once } -func lockRepo(ctx context.Context, repo restic.Repository) (*restic.Lock, context.Context, error) { - return lockRepository(ctx, repo, false) +func lockRepo(ctx context.Context, repo restic.Repository, retryLock time.Duration, json bool) (*restic.Lock, context.Context, error) { + return lockRepository(ctx, repo, false, retryLock, json) } -func lockRepoExclusive(ctx context.Context, repo restic.Repository) (*restic.Lock, context.Context, error) { - return lockRepository(ctx, repo, true) +func lockRepoExclusive(ctx context.Context, repo restic.Repository, retryLock time.Duration, json bool) (*restic.Lock, context.Context, error) { + return lockRepository(ctx, repo, true, retryLock, json) +} + +var ( + retrySleepStart = 5 * time.Second + retrySleepMax = 60 * time.Second +) + +func minDuration(a, b time.Duration) time.Duration { + if a <= b { + return a + } + return b } // lockRepository wraps the ctx such that it is cancelled when the repository is unlocked // cancelling the original context also stops the lock refresh -func lockRepository(ctx context.Context, repo restic.Repository, exclusive bool) (*restic.Lock, context.Context, error) { +func lockRepository(ctx context.Context, repo restic.Repository, exclusive bool, retryLock time.Duration, json bool) (*restic.Lock, context.Context, error) { // make sure that a repository is unlocked properly and after cancel() was // called by the cleanup handler in global.go globalLocks.Do(func() { @@ -43,7 +55,44 @@ func lockRepository(ctx context.Context, repo restic.Repository, exclusive bool) lockFn = restic.NewExclusiveLock } - lock, err := lockFn(ctx, repo) + var lock *restic.Lock + var err error + + retrySleep := minDuration(retrySleepStart, retryLock) + retryMessagePrinted := false + retryTimeout := time.After(retryLock) + +retryLoop: + for { + lock, err = lockFn(ctx, repo) + if err != nil && restic.IsAlreadyLocked(err) { + + if !retryMessagePrinted { + if !json { + Verbosef("repo already locked, waiting up to %s for the lock\n", retryLock) + } + retryMessagePrinted = true + } + + debug.Log("repo already locked, retrying in %v", retrySleep) + retrySleepCh := time.After(retrySleep) + + select { + case <-ctx.Done(): + return nil, ctx, ctx.Err() + case <-retryTimeout: + debug.Log("repo already locked, timeout expired") + // Last lock attempt + lock, err = lockFn(ctx, repo) + break retryLoop + case <-retrySleepCh: + retrySleep = minDuration(retrySleep*2, retrySleepMax) + } + } else { + // anything else, either a successful lock or another error + break retryLoop + } + } if restic.IsInvalidLock(err) { return nil, ctx, errors.Fatalf("%v\n\nthe `unlock --remove-all` command can be used to remove invalid locks. Make sure that no other restic process is accessing the repository when running the command", err) } diff --git a/cmd/restic/lock_test.go b/cmd/restic/lock_test.go index c074f15a6..86daf83f0 100644 --- a/cmd/restic/lock_test.go +++ b/cmd/restic/lock_test.go @@ -3,11 +3,14 @@ package main import ( "context" "fmt" + "runtime" + "strings" "testing" "time" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test" ) @@ -23,8 +26,8 @@ func openTestRepo(t *testing.T, wrapper backendWrapper) (*repository.Repository, return repo, cleanup, env } -func checkedLockRepo(ctx context.Context, t *testing.T, repo restic.Repository) (*restic.Lock, context.Context) { - lock, wrappedCtx, err := lockRepo(ctx, repo) +func checkedLockRepo(ctx context.Context, t *testing.T, repo restic.Repository, env *testEnvironment) (*restic.Lock, context.Context) { + lock, wrappedCtx, err := lockRepo(ctx, repo, env.gopts.RetryLock, env.gopts.JSON) rtest.OK(t, err) rtest.OK(t, wrappedCtx.Err()) if lock.Stale() { @@ -34,10 +37,10 @@ func checkedLockRepo(ctx context.Context, t *testing.T, repo restic.Repository) } func TestLock(t *testing.T) { - repo, cleanup, _ := openTestRepo(t, nil) + repo, cleanup, env := openTestRepo(t, nil) defer cleanup() - lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo) + lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, env) unlockRepo(lock) if wrappedCtx.Err() == nil { t.Fatal("unlock did not cancel context") @@ -45,12 +48,12 @@ func TestLock(t *testing.T) { } func TestLockCancel(t *testing.T) { - repo, cleanup, _ := openTestRepo(t, nil) + repo, cleanup, env := openTestRepo(t, nil) defer cleanup() ctx, cancel := context.WithCancel(context.Background()) defer cancel() - lock, wrappedCtx := checkedLockRepo(ctx, t, repo) + lock, wrappedCtx := checkedLockRepo(ctx, t, repo, env) cancel() if wrappedCtx.Err() == nil { t.Fatal("canceled parent context did not cancel context") @@ -61,10 +64,10 @@ func TestLockCancel(t *testing.T) { } func TestLockUnlockAll(t *testing.T) { - repo, cleanup, _ := openTestRepo(t, nil) + repo, cleanup, env := openTestRepo(t, nil) defer cleanup() - lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo) + lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, env) _, err := unlockAll(0) rtest.OK(t, err) if wrappedCtx.Err() == nil { @@ -81,10 +84,10 @@ func TestLockConflict(t *testing.T) { repo2, err := OpenRepository(context.TODO(), env.gopts) rtest.OK(t, err) - lock, _, err := lockRepoExclusive(context.Background(), repo) + lock, _, err := lockRepoExclusive(context.Background(), repo, env.gopts.RetryLock, env.gopts.JSON) rtest.OK(t, err) defer unlockRepo(lock) - _, _, err = lockRepo(context.Background(), repo2) + _, _, err = lockRepo(context.Background(), repo2, env.gopts.RetryLock, env.gopts.JSON) if err == nil { t.Fatal("second lock should have failed") } @@ -104,7 +107,7 @@ func (b *writeOnceBackend) Save(ctx context.Context, h restic.Handle, rd restic. } func TestLockFailedRefresh(t *testing.T) { - repo, cleanup, _ := openTestRepo(t, func(r restic.Backend) (restic.Backend, error) { + repo, cleanup, env := openTestRepo(t, func(r restic.Backend) (restic.Backend, error) { return &writeOnceBackend{Backend: r}, nil }) defer cleanup() @@ -117,7 +120,7 @@ func TestLockFailedRefresh(t *testing.T) { refreshInterval, refreshabilityTimeout = ri, rt }() - lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo) + lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, env) select { case <-wrappedCtx.Done(): @@ -136,11 +139,13 @@ type loggingBackend struct { func (b *loggingBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { b.t.Logf("save %v @ %v", h, time.Now()) - return b.Backend.Save(ctx, h, rd) + err := b.Backend.Save(ctx, h, rd) + b.t.Logf("save finished %v @ %v", h, time.Now()) + return err } func TestLockSuccessfulRefresh(t *testing.T) { - repo, cleanup, _ := openTestRepo(t, func(r restic.Backend) (restic.Backend, error) { + repo, cleanup, env := openTestRepo(t, func(r restic.Backend) (restic.Backend, error) { return &loggingBackend{ Backend: r, t: t, @@ -151,20 +156,99 @@ func TestLockSuccessfulRefresh(t *testing.T) { t.Logf("test for successful lock refresh %v", time.Now()) // reduce locking intervals to be suitable for testing ri, rt := refreshInterval, refreshabilityTimeout - refreshInterval = 40 * time.Millisecond - refreshabilityTimeout = 200 * time.Millisecond + refreshInterval = 60 * time.Millisecond + refreshabilityTimeout = 500 * time.Millisecond defer func() { refreshInterval, refreshabilityTimeout = ri, rt }() - lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo) + lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, env) select { case <-wrappedCtx.Done(): - t.Fatal("lock refresh failed") + // don't call t.Fatal to allow the lock to be properly cleaned up + t.Error("lock refresh failed", time.Now()) + + // Dump full stacktrace + buf := make([]byte, 1024*1024) + n := runtime.Stack(buf, true) + buf = buf[:n] + t.Log(string(buf)) + case <-time.After(2 * refreshabilityTimeout): // expected lock refresh to work } // unlockRepo should not crash unlockRepo(lock) } + +func TestLockWaitTimeout(t *testing.T) { + repo, cleanup, env := openTestRepo(t, nil) + defer cleanup() + + elock, _, err := lockRepoExclusive(context.TODO(), repo, env.gopts.RetryLock, env.gopts.JSON) + test.OK(t, err) + + retryLock := 200 * time.Millisecond + + start := time.Now() + lock, _, err := lockRepo(context.TODO(), repo, retryLock, env.gopts.JSON) + duration := time.Since(start) + + test.Assert(t, err != nil, + "create normal lock with exclusively locked repo didn't return an error") + test.Assert(t, strings.Contains(err.Error(), "repository is already locked exclusively"), + "create normal lock with exclusively locked repo didn't return the correct error") + test.Assert(t, retryLock <= duration && duration < retryLock*3/2, + "create normal lock with exclusively locked repo didn't wait for the specified timeout") + + test.OK(t, lock.Unlock()) + test.OK(t, elock.Unlock()) +} +func TestLockWaitCancel(t *testing.T) { + repo, cleanup, env := openTestRepo(t, nil) + defer cleanup() + + elock, _, err := lockRepoExclusive(context.TODO(), repo, env.gopts.RetryLock, env.gopts.JSON) + test.OK(t, err) + + retryLock := 200 * time.Millisecond + cancelAfter := 40 * time.Millisecond + + ctx, cancel := context.WithCancel(context.TODO()) + time.AfterFunc(cancelAfter, cancel) + + start := time.Now() + lock, _, err := lockRepo(ctx, repo, retryLock, env.gopts.JSON) + duration := time.Since(start) + + test.Assert(t, err != nil, + "create normal lock with exclusively locked repo didn't return an error") + test.Assert(t, strings.Contains(err.Error(), "context canceled"), + "create normal lock with exclusively locked repo didn't return the correct error") + test.Assert(t, cancelAfter <= duration && duration < retryLock-10*time.Millisecond, + "create normal lock with exclusively locked repo didn't return in time") + + test.OK(t, lock.Unlock()) + test.OK(t, elock.Unlock()) +} + +func TestLockWaitSuccess(t *testing.T) { + repo, cleanup, env := openTestRepo(t, nil) + defer cleanup() + + elock, _, err := lockRepoExclusive(context.TODO(), repo, env.gopts.RetryLock, env.gopts.JSON) + test.OK(t, err) + + retryLock := 200 * time.Millisecond + unlockAfter := 40 * time.Millisecond + + time.AfterFunc(unlockAfter, func() { + test.OK(t, elock.Unlock()) + }) + + lock, _, err := lockRepo(context.TODO(), repo, retryLock, env.gopts.JSON) + test.OK(t, err) + + test.OK(t, lock.Unlock()) +} diff --git a/doc/020_installation.rst b/doc/020_installation.rst index 5ae93c94d..4d591356d 100644 --- a/doc/020_installation.rst +++ b/doc/020_installation.rst @@ -40,7 +40,7 @@ package from the official community repos, e.g. using ``apk``: Arch Linux ========== -On `Arch Linux `__, there is a package called ``restic`` +On `Arch Linux `__, there is a package called ``restic`` installed from the official community repos, e.g. with ``pacman -S``: .. code-block:: console @@ -93,7 +93,7 @@ You may also install it using `MacPorts `__: Nix & NixOS =========== -If you are using `Nix `__ or `NixOS `__ +If you are using `Nix / NixOS `__ there is a package available named ``restic``. It can be installed using ``nix-env``: @@ -269,9 +269,10 @@ From Source *********** restic is written in the Go programming language and you need at least -Go version 1.18. Building restic may also work with older versions of Go, +Go version 1.18. Building for Solaris requires at least Go version 1.20. +Building restic may also work with older versions of Go, but that's not supported. See the `Getting -started `__ guide of the Go project for +started `__ guide of the Go project for instructions how to install Go. In order to build restic from source, execute the following steps: diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index 7484c0e5d..77df6bfc3 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -273,7 +273,7 @@ For an S3-compatible server that is not Amazon (like Minio, see below), or is only available via HTTP, you can specify the URL to the server like this: ``s3:http://server:port/bucket_name``. -.. note:: restic expects `path-style URLs `__ +.. note:: restic expects `path-style URLs `__ like for example ``s3.us-west-2.amazonaws.com/bucket_name``. Virtual-hosted–style URLs like ``bucket_name.s3.us-west-2.amazonaws.com``, where the bucket name is part of the hostname are not supported. These must @@ -290,12 +290,11 @@ like this: ``s3:http://server:port/bucket_name``. Minio Server ************ -`Minio `__ is an Open Source Object Storage, +`Minio `__ is an Open Source Object Storage, written in Go and compatible with Amazon S3 API. -- Download and Install `Minio - Server `__. -- You can also refer to https://docs.minio.io for step by step guidance +- Download and Install `Minio Download `__. +- You can also refer to `Minio Docs `__ for step by step guidance on installation and getting started on Minio Client and Minio Server. You must first setup the following environment variables with the @@ -350,7 +349,7 @@ this command. Alibaba Cloud (Aliyun) Object Storage System (OSS) ************************************************** -`Alibaba OSS `__ is an +`Alibaba OSS `__ is an encrypted, secure, cost-effective, and easy-to-use object storage service that enables you to store, back up, and archive large amounts of data in the cloud. @@ -358,7 +357,7 @@ of data in the cloud. Alibaba OSS is S3 compatible so it can be used as a storage provider for a restic repository with a couple of extra parameters. -- Determine the correct `Alibaba OSS region endpoint `__ - this will be something like ``oss-eu-west-1.aliyuncs.com`` +- Determine the correct `Alibaba OSS region endpoint `__ - this will be something like ``oss-eu-west-1.aliyuncs.com`` - You'll need the region name too - this will be something like ``oss-eu-west-1`` You must first setup the following environment variables with the @@ -441,7 +440,7 @@ the naming convention of those variables follows the official Python Swift clien Restic should be compatible with an `OpenStack RC file -`__ +`__ in most cases. Once environment variables are set up, a new repository can be created. The @@ -614,9 +613,9 @@ The number of concurrent connections to the GCS service can be set with the ``-o gs.connections=10`` switch. By default, at most five parallel connections are established. -.. _service account: https://cloud.google.com/iam/docs/service-accounts -.. _create a service account key: https://cloud.google.com/iam/docs/creating-managing-service-account-keys#iam-service-account-keys-create-console -.. _default authentication material: https://cloud.google.com/docs/authentication/production +.. _service account: https://cloud.google.com/iam/docs/service-account-overview +.. _create a service account key: https://cloud.google.com/iam/docs/keys-create-delete +.. _default authentication material: https://cloud.google.com/docs/authentication#service-accounts .. _other-services: @@ -776,7 +775,7 @@ Password prompt on Windows At the moment, restic only supports the default Windows console interaction. If you use emulation environments like -`MSYS2 `__ or +`MSYS2 `__ or `Cygwin `__, which use terminals like ``Mintty`` or ``rxvt``, you may get a password error. diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 21b357981..39d753f54 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -225,7 +225,7 @@ the exclude options are: - ``--exclude`` Specified one or more times to exclude one or more items - ``--iexclude`` Same as ``--exclude`` but ignores the case of paths -- ``--exclude-caches`` Specified once to exclude folders containing `this special file `__ +- ``--exclude-caches`` Specified once to exclude a folder's content if it contains `the special CACHEDIR.TAG file `__, but keep ``CACHEDIR.TAG``. - ``--exclude-file`` Specified one or more times to exclude items listed in a given file - ``--iexclude-file`` Same as ``exclude-file`` but ignores cases like in ``--iexclude`` - ``--exclude-if-present foo`` Specified one or more times to exclude a folder's content if it contains a file called ``foo`` (optionally having a given header, no wildcards for the file name supported) @@ -254,14 +254,14 @@ This instructs restic to exclude files matching the following criteria: * All files matching ``*.go`` (second line in ``excludes.txt``) * All files and sub-directories named ``bar`` which reside somewhere below a directory called ``foo`` (fourth line in ``excludes.txt``) -Patterns use `filepath.Glob `__ internally, -see `filepath.Match `__ for -syntax. Patterns are tested against the full path of a file/dir to be saved, +Patterns use the syntax of the Go function +`filepath.Match `__ +and are tested against the full path of a file/dir to be saved, even if restic is passed a relative path to save. Empty lines and lines starting with a ``#`` are ignored. Environment variables in exclude files are expanded with `os.ExpandEnv -`__, so ``/home/$USER/foo`` will be +`__, so ``/home/$USER/foo`` will be expanded to ``/home/bob/foo`` for the user ``bob``. To get a literal dollar sign, write ``$$`` to the file - this has to be done even when there's no matching environment variable for the word following a single ``$``. Note @@ -381,7 +381,7 @@ contains one *pattern* per line. The file must be encoded as UTF-8, or UTF-16 with a byte-order mark. Leading and trailing whitespace is removed from the patterns. Empty lines and lines starting with a ``#`` are ignored and each pattern is expanded when read, such that special characters in it are expanded -using the Go function `filepath.Glob `__ +using the Go function `filepath.Glob `__ - please see its documentation for the syntax you can use in the patterns. The argument passed to ``--files-from-verbatim`` must be the name of a text file @@ -533,8 +533,11 @@ Restic does not have a built-in way of scheduling backups, as it's a tool that runs when executed rather than a daemon. There are plenty of different ways to schedule backup runs on various different platforms, e.g. systemd and cron on Linux/BSD and Task Scheduler in Windows, depending on one's -needs and requirements. When scheduling restic to run recurringly, please -make sure to detect already running instances before starting the backup. +needs and requirements. If you don't want to implement your own scheduling, +you can use `resticprofile `__. + +When scheduling restic to run recurringly, please make sure to detect already +running instances before starting the backup. Space requirements ****************** diff --git a/doc/060_forget.rst b/doc/060_forget.rst index b960ddb14..72c7ae97f 100644 --- a/doc/060_forget.rst +++ b/doc/060_forget.rst @@ -205,6 +205,7 @@ The ``forget`` command accepts the following policy options: natural time boundaries and *not* relative to when you run ``forget``. Weeks are Monday 00:00 to Sunday 23:59, days 00:00 to 23:59, hours :00 to :59, etc. They also only count hours/days/weeks/etc which have one or more snapshots. + A value of ``-1`` will be interpreted as "forever", i.e. "keep all". .. note:: All duration related options (``--keep-{within,-*}``) ignore snapshots with a timestamp in the future (relative to when the ``forget`` command is @@ -471,7 +472,7 @@ space. However, a **failed** ``prune`` run can cause the repository to become **temporarily unusable**. Therefore, make sure that you have a stable connection to the repository storage, before running this command. In case the command fails, it may become necessary to manually remove all files from the `index/` folder of the repository and -run `rebuild-index` afterwards. +run `repair index` afterwards. To prevent accidental usages of the ``--unsafe-recover-no-free-space`` option it is necessary to first run ``prune --unsafe-recover-no-free-space SOME-ID`` and then replace diff --git a/doc/070_encryption.rst b/doc/070_encryption.rst index a7b8716ac..dc651cc07 100644 --- a/doc/070_encryption.rst +++ b/doc/070_encryption.rst @@ -19,7 +19,7 @@ Encryption the implementation looks sane and I guess the deduplication trade-off is worth it. So… I’m going to use restic for my personal backups.*" `Filippo Valsorda`_ -.. _Filippo Valsorda: https://blog.filippo.io/restic-cryptography/ +.. _Filippo Valsorda: https://words.filippo.io/restic-cryptography/ ********************** Manage repository keys diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index 712a70244..dc7f782dc 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -22,18 +22,18 @@ Check if a repository is already initialized You may find a need to check if a repository is already initialized, perhaps to prevent your script from initializing a repository multiple -times. The command ``snapshots`` may be used for this purpose: +times. The command ``cat config`` may be used for this purpose: .. code-block:: console - $ restic -r /srv/restic-repo snapshots - Fatal: unable to open config file: Stat: stat /srv/restic-repo/config: no such file or directory + $ restic -r /srv/restic-repo cat config + Fatal: unable to open config file: stat /srv/restic-repo/config: no such file or directory Is there a repository at the following location? /srv/restic-repo If a repository does not exist, restic will return a non-zero exit code and print an error message. Note that restic will also return a non-zero exit code if a different error is encountered (e.g.: incorrect password -to ``snapshots``) and it may print a different error message. If there -are no errors, restic will return a zero exit code and print all the -snapshots. +to ``cat config``) and it may print a different error message. If there +are no errors, restic will return a zero exit code and print the repository +metadata. diff --git a/doc/077_troubleshooting.rst b/doc/077_troubleshooting.rst new file mode 100644 index 000000000..fe317acfc --- /dev/null +++ b/doc/077_troubleshooting.rst @@ -0,0 +1,194 @@ +.. + Normally, there are no heading levels assigned to certain characters as the structure is + determined from the succession of headings. However, this convention is used in Python’s + Style Guide for documenting which you may follow: + + # with overline, for parts + * for chapters + = for sections + - for subsections + ^ for subsubsections + " for paragraphs + +######################### +Troubleshooting +######################### + +The repository format used by restic is designed to be error resistant. In +particular, commands like, for example, ``backup`` or ``prune`` can be interrupted +at *any* point in time without damaging the repository. You might have to run +``unlock`` manually though, but that's it. + +However, a repository might be damaged if some of its files are damaged or lost. +This can occur due to hardware failures, accidentally removing files from the +repository or bugs in the implementation of restic. + +The following steps will help you recover a repository. This guide does not cover +all possible types of repository damages. Thus, if the steps do not work for you +or you are unsure how to proceed, then ask for help. Please always include the +check output discussed in the next section and what steps you've taken to repair +the repository so far. + +* `Forum `_ +* Our IRC channel ``#restic`` on ``irc.libera.chat`` + +Make sure that you **use the latest available restic version**. It can contain +bugfixes, and improvements to simplify the repair of a repository. It might also +contain a fix for your repository problems! + + +1. Find out what is damaged +*************************** + +The first step is always to check the repository. + +.. code-block:: console + + $ restic check --read-data + + using temporary cache in /tmp/restic-check-cache-1418935501 + repository 12345678 opened (version 2, compression level auto) + created new cache in /tmp/restic-check-cache-1418935501 + create exclusive lock for repository + load indexes + check all packs + check snapshots, trees and blobs + error for tree 7ef8ebab: + id 7ef8ebabc59aadda1a237d23ca7abac487b627a9b86508aa0194690446ff71f6 not found in repository + [0:02] 100.00% 7 / 7 snapshots + read all data + [0:05] 100.00% 25 / 25 packs + Fatal: repository contains errors + +.. note:: + + This will download the whole repository. If retrieving data from the backend is + expensive, then omit the ``--read-data`` option. Keep a copy of the check output + as it might be necessary later on! + +If the output contains warnings that the ``ciphertext verification failed`` for +some blobs in the repository, then please ask for help in the forum or our IRC +channel. These errors are often caused by hardware problems which **must** be +investigated and fixed. Otherwise, the backup will be damaged again and again. + +Similarly, if a repository is repeatedly damaged, please open an `issue on Github +`_ as this could indicate a bug +somewhere. Please include the check output and additional information that might +help locate the problem. + + +2. Backup the repository +************************ + +Create a full copy of the repository if possible. Or at the very least make a +copy of the ``index`` and ``snapshots`` folders. This will allow you to roll back +the repository if the repair procedure fails. If your repository resides in a +cloud storage, then you can for example use `rclone `_ to +make such a copy. + +Please disable all regular operations on the repository to prevent unexpected +changes. Especially, ``forget`` or ``prune`` must be disabled as they could +remove data unexpectedly. + +.. warning:: + + If you suspect hardware problems, then you *must* investigate those first. + Otherwise, the repository will soon be damaged again. + +Please take the time to understand what the commands described in the following +do. If you are unsure, then ask for help in the forum or our IRC channel. Search +whether your issue is already known and solved. Please take a look at the +`forum`_ and `Github issues `_. + + +3. Repair the index +******************* + +Restic relies on its index to contain correct information about what data is +stored in the repository. Thus, the first step to repair a repository is to +repair the index: + +.. code-block:: console + + $ restic repair index + + repository a14e5863 opened (version 2, compression level auto) + loading indexes... + getting pack files to read... + removing not found pack file 83ad44f59b05f6bce13376b022ac3194f24ca19e7a74926000b6e316ec6ea5a4 + rebuilding index + [0:00] 100.00% 27 / 27 packs processed + deleting obsolete index files + [0:00] 100.00% 3 / 3 files deleted + done + +This ensures that no longer existing files are removed from the index. All later +steps to repair the repository rely on a correct index. That is, you must always +repair the index first! + +Please note that it is not recommended to repair the index unless the repository +is actually damaged. + + +4. Run all backups (optional) +***************************** + +With a correct index, the ``backup`` command guarantees that newly created +snapshots can be restored successfully. It can also heal older snapshots, +if the missing data is also contained in the new snapshot. + +Therefore, it is recommended to run all your ``backup`` tasks again. In some +cases, this is enough to fully repair the repository. + + +5. Remove missing data from snapshots +************************************* + +If your repository is still missing data, then you can use the ``repair snapshots`` +command to remove all inaccessible data from the snapshots. That is, this will +result in a limited amount of data loss. Using the ``--forget`` option, the +command will automatically remove the original, damaged snapshots. + +.. code-block:: console + + $ restic repair snapshots --forget + + snapshot 6979421e of [/home/user/restic/restic] at 2022-11-02 20:59:18.617503315 +0100 CET) + file "/restic/internal/fuse/snapshots_dir.go": removed missing content + file "/restic/internal/restorer/restorer_unix_test.go": removed missing content + file "/restic/internal/walker/walker.go": removed missing content + saved new snapshot 7b094cea + removed old snapshot 6979421e + + modified 1 snapshots + +If you did not add the ``--forget`` option, then you have to manually delete all +modified snapshots using the ``forget`` command. In the example above, you'd have +to run ``restic forget 6979421e``. + + +6. Check the repository again +***************************** + +Phew, we're almost done now. To make sure that the repository has been successfully +repaired please run ``check`` again. + +.. code-block:: console + + $ restic check --read-data + + using temporary cache in /tmp/restic-check-cache-2569290785 + repository a14e5863 opened (version 2, compression level auto) + created new cache in /tmp/restic-check-cache-2569290785 + create exclusive lock for repository + load indexes + check all packs + check snapshots, trees and blobs + [0:00] 100.00% 7 / 7 snapshots + read all data + [0:00] 100.00% 25 / 25 packs + no errors were found + +If the ``check`` command did not complete with ``no errors were found``, then +the repository is still damaged. At this point, please ask for help at the +`forum`_ or our IRC channel ``#restic`` on ``irc.libera.chat``. diff --git a/doc/090_participating.rst b/doc/090_participating.rst index 00a387974..890bd9018 100644 --- a/doc/090_participating.rst +++ b/doc/090_participating.rst @@ -33,8 +33,8 @@ The debug log will always contain all log messages restic generates. You can also instruct restic to print some or all debug messages to stderr. These can also be limited to e.g. a list of source files or a list of patterns for function names. The patterns are globbing patterns (see the -documentation for `path.Glob `__), multiple -patterns are separated by commas. Patterns are case sensitive. +documentation for `filepath.Match `__). +Multiple patterns are separated by commas. Patterns are case sensitive. Printing all log messages to the console can be achieved by setting the file filter to ``*``: diff --git a/doc/110_talks.rst b/doc/110_talks.rst index 06952896f..e32cda62a 100644 --- a/doc/110_talks.rst +++ b/doc/110_talks.rst @@ -17,6 +17,8 @@ Talks The following talks will be or have been given about restic: +- 2021-04-02: `The Changelog: Restic has your backup + (Podcast) `__ - 2016-01-31: Lightning Talk at the Go Devroom at FOSDEM 2016, Brussels, Belgium - 2016-01-29: `restic - Backups mal @@ -24,11 +26,11 @@ The following talks will be or have been given about restic: Public lecture in German at `CCC Cologne e.V. `__ in Cologne, Germany - 2015-08-23: `A Solution to the Backup - Inconvenience `__: - Lecture at `FROSCON 2015 `__ in Bonn, Germany + Inconvenience `__: + Lecture at `FROSCON 2015 `__ in Bonn, Germany - 2015-02-01: `Lightning Talk at FOSDEM 2015 `__: A short introduction (with slightly outdated command line) - 2015-01-27: `Talk about restic at CCC - Aachen `__ + Aachen `__ (in German) diff --git a/doc/design.rst b/doc/design.rst index 7102585ac..94dabdc34 100644 --- a/doc/design.rst +++ b/doc/design.rst @@ -603,7 +603,10 @@ that the process is dead and considers the lock to be stale. When a new lock is to be created and no other conflicting locks are detected, restic creates a new lock, waits, and checks if other locks appeared in the repository. Depending on the type of the other locks and -the lock to be created, restic either continues or fails. +the lock to be created, restic either continues or fails. If the +``--retry-lock`` option is specified, restic will retry +creating the lock periodically until it succeeds or the specified +timeout expires. Read and Write Ordering ======================= diff --git a/doc/developer_information.rst b/doc/developer_information.rst index c05edc9d2..307851757 100644 --- a/doc/developer_information.rst +++ b/doc/developer_information.rst @@ -10,7 +10,7 @@ refer to the documentation for the respective version. The binary produced depends on the following things: * The source code for the release - * The exact version of the official `Go compiler `__ used to produce the binaries (running ``restic version`` will print this) + * The exact version of the official `Go compiler `__ used to produce the binaries (running ``restic version`` will print this) * The architecture and operating system the Go compiler runs on (Linux, ``amd64``) * The build tags (for official binaries, it's the tag ``selfupdate``) * The path where the source code is extracted to (``/restic``) diff --git a/doc/index.rst b/doc/index.rst index 034dbda23..8b72dcf58 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -14,6 +14,7 @@ Restic Documentation 060_forget 070_encryption 075_scripting + 077_troubleshooting 080_examples 090_participating 100_references diff --git a/doc/man/restic-backup.1 b/doc/man/restic-backup.1 index 2598678d0..4297c3b8e 100644 --- a/doc/man/restic-backup.1 +++ b/doc/man/restic-backup.1 @@ -205,7 +205,7 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-cache.1 b/doc/man/restic-cache.1 index 302bcb01e..3552fb1dc 100644 --- a/doc/man/restic-cache.1 +++ b/doc/man/restic-cache.1 @@ -118,7 +118,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-cat.1 b/doc/man/restic-cat.1 index 1fb7dd45f..2e787fa06 100644 --- a/doc/man/restic-cat.1 +++ b/doc/man/restic-cat.1 @@ -106,7 +106,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-check.1 b/doc/man/restic-check.1 index e08b83d46..e641fc2b5 100644 --- a/doc/man/restic-check.1 +++ b/doc/man/restic-check.1 @@ -123,7 +123,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-copy.1 b/doc/man/restic-copy.1 index 07dcfe957..53badecc9 100644 --- a/doc/man/restic-copy.1 +++ b/doc/man/restic-copy.1 @@ -147,7 +147,7 @@ new destination repository using the "init" command. .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-diff.1 b/doc/man/restic-diff.1 index f0707a257..31c34dc8a 100644 --- a/doc/man/restic-diff.1 +++ b/doc/man/restic-diff.1 @@ -126,7 +126,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-dump.1 b/doc/man/restic-dump.1 index f9a2368bc..61b3b3ec8 100644 --- a/doc/man/restic-dump.1 +++ b/doc/man/restic-dump.1 @@ -129,7 +129,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-find.1 b/doc/man/restic-find.1 index 4f5bdd4e3..9fa4dd811 100644 --- a/doc/man/restic-find.1 +++ b/doc/man/restic-find.1 @@ -151,7 +151,7 @@ It can also be used to search for restic blobs or trees for troubleshooting. .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH EXAMPLE diff --git a/doc/man/restic-forget.1 b/doc/man/restic-forget.1 index f46d05736..d8a69856e 100644 --- a/doc/man/restic-forget.1 +++ b/doc/man/restic-forget.1 @@ -217,7 +217,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-generate.1 b/doc/man/restic-generate.1 index e3733ce60..6b54ebfca 100644 --- a/doc/man/restic-generate.1 +++ b/doc/man/restic-generate.1 @@ -127,7 +127,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-init.1 b/doc/man/restic-init.1 index 80edf5362..194f31756 100644 --- a/doc/man/restic-init.1 +++ b/doc/man/restic-init.1 @@ -134,7 +134,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-key.1 b/doc/man/restic-key.1 index ff6ab4fd0..4163cefa5 100644 --- a/doc/man/restic-key.1 +++ b/doc/man/restic-key.1 @@ -118,7 +118,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-list.1 b/doc/man/restic-list.1 index e2f878c76..6683e2c47 100644 --- a/doc/man/restic-list.1 +++ b/doc/man/restic-list.1 @@ -106,7 +106,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-ls.1 b/doc/man/restic-ls.1 index afd72ff71..a16716434 100644 --- a/doc/man/restic-ls.1 +++ b/doc/man/restic-ls.1 @@ -141,7 +141,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-migrate.1 b/doc/man/restic-migrate.1 index ee4d44e71..d8127090e 100644 --- a/doc/man/restic-migrate.1 +++ b/doc/man/restic-migrate.1 @@ -112,7 +112,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-mount.1 b/doc/man/restic-mount.1 index da38ae451..ce4f893a7 100644 --- a/doc/man/restic-mount.1 +++ b/doc/man/restic-mount.1 @@ -190,7 +190,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-prune.1 b/doc/man/restic-prune.1 index 88c03f72a..197cb1130 100644 --- a/doc/man/restic-prune.1 +++ b/doc/man/restic-prune.1 @@ -135,7 +135,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-rebuild-index.1 b/doc/man/restic-rebuild-index.1 index 3be67e79e..18878b66f 100644 --- a/doc/man/restic-rebuild-index.1 +++ b/doc/man/restic-rebuild-index.1 @@ -111,7 +111,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-recover.1 b/doc/man/restic-recover.1 index 7415a1113..aa3441156 100644 --- a/doc/man/restic-recover.1 +++ b/doc/man/restic-recover.1 @@ -108,7 +108,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-restore.1 b/doc/man/restic-restore.1 index 2348f7478..39ff62059 100644 --- a/doc/man/restic-restore.1 +++ b/doc/man/restic-restore.1 @@ -151,7 +151,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-rewrite.1 b/doc/man/restic-rewrite.1 index 9f33bcb64..6edf51b95 100644 --- a/doc/man/restic-rewrite.1 +++ b/doc/man/restic-rewrite.1 @@ -159,7 +159,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-self-update.1 b/doc/man/restic-self-update.1 index 25f863396..e311b2277 100644 --- a/doc/man/restic-self-update.1 +++ b/doc/man/restic-self-update.1 @@ -113,7 +113,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-snapshots.1 b/doc/man/restic-snapshots.1 index 78cd664e3..d2dbf52ee 100644 --- a/doc/man/restic-snapshots.1 +++ b/doc/man/restic-snapshots.1 @@ -130,7 +130,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-stats.1 b/doc/man/restic-stats.1 index 6e3b9838b..694bde22d 100644 --- a/doc/man/restic-stats.1 +++ b/doc/man/restic-stats.1 @@ -152,7 +152,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-tag.1 b/doc/man/restic-tag.1 index 06bf25495..1ff0b4f78 100644 --- a/doc/man/restic-tag.1 +++ b/doc/man/restic-tag.1 @@ -137,7 +137,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-unlock.1 b/doc/man/restic-unlock.1 index c4ad7f050..e5b408915 100644 --- a/doc/man/restic-unlock.1 +++ b/doc/man/restic-unlock.1 @@ -110,7 +110,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-version.1 b/doc/man/restic-version.1 index b410d1231..eca34d60a 100644 --- a/doc/man/restic-version.1 +++ b/doc/man/restic-version.1 @@ -107,7 +107,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic.1 b/doc/man/restic.1 index 76602d02d..f76d16e38 100644 --- a/doc/man/restic.1 +++ b/doc/man/restic.1 @@ -100,7 +100,7 @@ directories in an encrypted repository stored on different backends. .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index 97480db80..093144722 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -26,7 +26,7 @@ Usage help is available: dump Print a backed-up file to stdout find Find a file, a directory or restic IDs forget Remove snapshots from the repository - generate Generate manual pages and auto-completion files (bash, fish, zsh) + generate Generate manual pages and auto-completion files (bash, fish, zsh, powershell) help Help about any command init Initialize a new repository key Manage keys (passwords) @@ -35,8 +35,8 @@ Usage help is available: migrate Apply migrations mount Mount the repository prune Remove unneeded data from the repository - rebuild-index Build a new index recover Recover data from the repository not referenced by snapshots + repair Repair the repository restore Extract the data from a snapshot rewrite Rewrite snapshots to exclude unwanted files self-update Update the restic binary @@ -50,7 +50,7 @@ Usage help is available: --cacert file file to load root certificates from (default: use system certificates) --cache-dir directory set the cache directory. (default: use system default cache directory) --cleanup-cache auto remove old cache directories - --compression mode compression mode (only available for repository format version 2), one of (auto|off|max) (default auto) + --compression mode compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) (default auto) -h, --help help for restic --insecure-tls skip TLS certificate verification when connecting to the repository (insecure) --json set output mode to JSON for commands that support it @@ -66,6 +66,7 @@ Usage help is available: -q, --quiet do not output comprehensive progress report -r, --repo repository repository to backup to or restore from (default: $RESTIC_REPOSITORY) --repository-file file file to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + --retry-lock duration retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) --tls-client-cert file path to a file containing PEM encoded TLS client certificate and private key -v, --verbose be verbose (specify multiple times or a level using --verbose=n, max level/times is 2) @@ -105,6 +106,7 @@ command: --files-from-raw file read the files to backup from file (can be combined with file args; can be specified multiple times) --files-from-verbatim file read the files to backup from file (can be combined with file args; can be specified multiple times) -f, --force force re-reading the target files/directories (overrides the "parent" flag) + -g, --group-by group group snapshots by host, paths and/or tags, separated by comma (disable grouping with '') (default host,paths) -h, --help help for backup -H, --host hostname set the hostname for the snapshot manually. To prevent an expensive rescan use the "parent" flag --iexclude pattern same as --exclude pattern but ignores the casing of filenames @@ -113,8 +115,8 @@ command: --ignore-inode ignore inode number changes when checking for modified files --no-scan do not run scanner to estimate size of backup -x, --one-file-system exclude other file systems, don't cross filesystem boundaries and subvolumes - --parent snapshot use this parent snapshot (default: last snapshot in the repository that has the same target files/directories, and is not newer than the snapshot time) - --read-concurrency n read n file concurrently (default: $RESTIC_READ_CONCURRENCY or 2) + --parent snapshot use this parent snapshot (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time) + --read-concurrency n read n files concurrently (default: $RESTIC_READ_CONCURRENCY or 2) --stdin read backup from stdin --stdin-filename filename filename to use when reading from stdin (default "stdin") --tag tags add tags for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times) (default []) @@ -126,7 +128,7 @@ command: --cacert file file to load root certificates from (default: use system certificates) --cache-dir directory set the cache directory. (default: use system default cache directory) --cleanup-cache auto remove old cache directories - --compression mode compression mode (only available for repository format version 2), one of (auto|off|max) (default auto) + --compression mode compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) (default auto) --insecure-tls skip TLS certificate verification when connecting to the repository (insecure) --json set output mode to JSON for commands that support it --key-hint key key ID of key to try decrypting first (default: $RESTIC_KEY_HINT) @@ -141,6 +143,7 @@ command: -q, --quiet do not output comprehensive progress report -r, --repo repository repository to backup to or restore from (default: $RESTIC_REPOSITORY) --repository-file file file to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + --retry-lock duration retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) --tls-client-cert file path to a file containing PEM encoded TLS client certificate and private key -v, --verbose be verbose (specify multiple times or a level using --verbose=n, max level/times is 2) @@ -224,7 +227,7 @@ locks with the following command: d369ccc7d126594950bf74f0a348d5d98d9e99f3215082eb69bf02dc9b3e464c The ``find`` command searches for a given -`pattern `__ in the +`pattern `__ in the repository. .. code-block:: console diff --git a/docker/Dockerfile b/docker/Dockerfile index 9f47fa10f..72fc85093 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,7 +11,7 @@ RUN go run build.go FROM alpine:latest AS restic -RUN apk add --update --no-cache ca-certificates fuse openssh-client tzdata +RUN apk add --update --no-cache ca-certificates fuse openssh-client tzdata jq COPY --from=builder /go/src/github.com/restic/restic/restic /usr/bin diff --git a/go.mod b/go.mod index 3271cf230..6806cb0f4 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,9 @@ module github.com/restic/restic require ( - cloud.google.com/go/storage v1.29.0 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.1 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 + cloud.google.com/go/storage v1.30.1 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 github.com/anacrolix/fuse v0.2.0 github.com/cenkalti/backoff/v4 v4.2.0 github.com/cespare/xxhash/v2 v2.2.0 @@ -14,9 +14,9 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/hirochachacha/go-smb2 v1.1.0 github.com/juju/ratelimit v1.0.2 - github.com/klauspost/compress v1.16.0 + github.com/klauspost/compress v1.16.5 github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5 - github.com/minio/minio-go/v7 v7.0.49 + github.com/minio/minio-go/v7 v7.0.52 github.com/minio/sha256-simd v1.0.0 github.com/ncw/swift/v2 v2.0.1 github.com/pkg/errors v0.9.1 @@ -26,21 +26,21 @@ require ( github.com/restic/chunker v0.4.0 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 - golang.org/x/crypto v0.7.0 - golang.org/x/net v0.8.0 - golang.org/x/oauth2 v0.5.0 + golang.org/x/crypto v0.8.0 + golang.org/x/net v0.9.0 + golang.org/x/oauth2 v0.7.0 golang.org/x/sync v0.1.0 - golang.org/x/sys v0.6.0 - golang.org/x/term v0.6.0 - golang.org/x/text v0.8.0 - google.golang.org/api v0.111.0 + golang.org/x/sys v0.7.0 + golang.org/x/term v0.7.0 + golang.org/x/text v0.9.0 + google.golang.org/api v0.116.0 ) require ( - cloud.google.com/go v0.108.0 // indirect - cloud.google.com/go/compute v1.18.0 // indirect + cloud.google.com/go v0.110.0 // indirect + cloud.google.com/go/compute v1.19.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.11.0 // indirect + cloud.google.com/go/iam v0.13.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/dnaeon/go-vcr v1.2.0 // indirect @@ -48,13 +48,13 @@ require ( github.com/felixge/fgprof v0.9.3 // indirect github.com/geoffgarside/ber v1.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.7.0 // indirect + github.com/googleapis/gax-go/v2 v2.8.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.3 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/kr/fs v0.1.0 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -65,9 +65,9 @@ require ( go.opencensus.io v0.24.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230223222841-637eb2293923 // indirect - google.golang.org/grpc v1.53.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 // indirect + google.golang.org/grpc v1.54.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 878707452..0e0dbf67a 100644 --- a/go.sum +++ b/go.sum @@ -1,22 +1,22 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.108.0 h1:xntQwnfn8oHGX0crLVinvHM+AhXvi3QHQIEcX/2hiWk= -cloud.google.com/go v0.108.0/go.mod h1:lNUfQqusBJp0bgAg6qrHgYFYbTB+dOiob1itwnlD33Q= -cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= -cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go/compute v1.19.0 h1:+9zda3WGgW1ZSTlVppLCYFIr48Pa35q1uG2N1itbCEQ= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/iam v0.11.0 h1:kwCWfKwB6ePZoZnGLwrd3B6Ru/agoHANTUBWpVNIdnM= -cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= -cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= -cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= -cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.1 h1:gVXuXcWd1i4C2Ruxe321aU+IKGaStvGB/S90PUPB/W8= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.1/go.mod h1:DffdKW9RFqa5VgmsjUOsS7UE7eiA5iAvYUs63bhKQ0M= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= +cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= +cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJVR2cMC8lvZcimipiEY= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= @@ -72,8 +72,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -84,7 +84,7 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b h1:8htHrh2bw9c7Idkb7YNac+ZpTqLMjRpI+FWu51ltaQc= github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= @@ -93,8 +93,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= -github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= -github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.8.0 h1:UBtEZqx1bjXtOQ5BVTkuYghXrr3N4V123VKJK67vJZc= +github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI= @@ -107,12 +107,12 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI= github.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= -github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= -github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= -github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5 h1:OUlGa6AAolmjyPtILbMJ8vHayz5wd4wBUloheGcMhfA= @@ -120,8 +120,8 @@ github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5/go.mod h1:4FCXMUWo9 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.49 h1:dE5DfOtnXMXCjr/HWI6zN9vCrY6Sv666qhhiwUMvGV4= -github.com/minio/minio-go/v7 v7.0.49/go.mod h1:UI34MvQEiob3Cf/gGExGMmzugkM/tNgbFypNDy5LMVc= +github.com/minio/minio-go/v7 v7.0.52 h1:8XhG36F6oKQUDDSuz6dY3rioMzovKjW40W6ANuN0Dps= +github.com/minio/minio-go/v7 v7.0.52/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -177,8 +177,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -194,11 +194,11 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= -golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -219,17 +219,17 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -242,8 +242,8 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/api v0.111.0 h1:bwKi+z2BsdwYFRKrqwutM+axAlYLz83gt5pDSXCJT+0= -google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0= +google.golang.org/api v0.116.0 h1:09tOPVufPwfm5W4aA8EizGHJ7BcoRDsIareM2a15gO4= +google.golang.org/api v0.116.0/go.mod h1:9cD4/t6uvd9naoEJFA+M96d0IuB6BqFuyhpw68+mRGg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= @@ -251,15 +251,15 @@ google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230223222841-637eb2293923 h1:znp6mq/drrY+6khTAlJUDNFFcDGV2ENLYKpMq8SyCds= -google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 h1:0BOZf6qNozI3pkN3fJLwNubheHJYHhMh91GRFOWWK08= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -271,8 +271,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index a56965d63..3c1cc33d0 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -207,7 +207,7 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error { if arch.Repo.Index().Has(restic.BlobHandle{ID: id, Type: restic.TreeBlob}) { err = errors.Errorf("tree %v could not be loaded; the repository could be damaged: %v", id, err) } else { - err = errors.Errorf("tree %v is not known; the repository could be damaged, run `rebuild-index` to try to repair it", id) + err = errors.Errorf("tree %v is not known; the repository could be damaged, run `repair index` to try to repair it", id) } return err } diff --git a/internal/archiver/tree_saver.go b/internal/archiver/tree_saver.go index d25781b03..a7dae3873 100644 --- a/internal/archiver/tree_saver.go +++ b/internal/archiver/tree_saver.go @@ -105,14 +105,15 @@ func (s *TreeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, I continue } - debug.Log("insert %v", fnr.node.Name) err := builder.AddNode(fnr.node) if err != nil && errors.Is(err, restic.ErrTreeNotOrdered) && lastNode != nil && fnr.node.Equals(*lastNode) { + debug.Log("insert %v failed: %v", fnr.node.Name, err) // ignore error if an _identical_ node already exists, but nevertheless issue a warning _ = s.errFn(fnr.target, err) err = nil } if err != nil { + debug.Log("insert %v failed: %v", fnr.node.Name, err) return nil, stats, err } lastNode = fnr.node diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index 02433795b..9a3695f0f 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -14,7 +14,6 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" - "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -26,7 +25,6 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" azContainer "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" - "github.com/cenkalti/backoff/v4" ) // Backend stores data on an azure endpoint. @@ -34,7 +32,6 @@ type Backend struct { cfg Config container *azContainer.Client connections uint - sem sema.Semaphore prefix string listMaxItems int layout.Layout @@ -96,16 +93,10 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { return nil, errors.New("no azure authentication information found") } - sem, err := sema.New(cfg.Connections) - if err != nil { - return nil, err - } - be := &Backend{ container: client, cfg: cfg, connections: cfg.Connections, - sem: sem, Layout: &layout.DefaultLayout{ Path: cfg.Prefix, Join: path.Join, @@ -152,7 +143,6 @@ func (be *Backend) SetListMaxItems(i int) { // IsNotExist returns true if the error is caused by a not existing file. func (be *Backend) IsNotExist(err error) bool { - debug.Log("IsNotExist(%T, %#v)", err, err) return bloberror.HasCode(err, bloberror.BlobNotFound) } @@ -187,16 +177,8 @@ func (be *Backend) Path() string { // Save stores data in the backend at the handle. func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { - if err := h.Valid(); err != nil { - return backoff.Permanent(err) - } - objName := be.Filename(h) - debug.Log("Save %v at %v", h, objName) - - be.sem.GetToken() - debug.Log("InsertObject(%v, %v)", be.cfg.AccountName, objName) var err error @@ -208,9 +190,6 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe err = be.saveLarge(ctx, objName, rd) } - be.sem.ReleaseToken() - debug.Log("%v, err %#v", objName, err) - return err } @@ -228,7 +207,7 @@ func (be *Backend) saveSmall(ctx context.Context, objName string, rd restic.Rewi reader := bytes.NewReader(buf) _, err = blockBlobClient.StageBlock(ctx, id, streaming.NopCloser(reader), &blockblob.StageBlockOptions{ - TransactionalContentMD5: rd.Hash(), + TransactionalValidation: blob.TransferValidationTypeMD5(rd.Hash()), }) if err != nil { return errors.Wrap(err, "StageBlock") @@ -271,7 +250,7 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi reader := bytes.NewReader(buf) debug.Log("StageBlock %v with %d bytes", id, len(buf)) _, err = blockBlobClient.StageBlock(ctx, id, streaming.NopCloser(reader), &blockblob.StageBlockOptions{ - TransactionalContentMD5: h[:], + TransactionalValidation: blob.TransferValidationTypeMD5(h[:]), }) if err != nil { @@ -299,23 +278,9 @@ func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset } func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h)) - if err := h.Valid(); err != nil { - return nil, backoff.Permanent(err) - } - - if offset < 0 { - return nil, errors.New("offset is negative") - } - - if length < 0 { - return nil, errors.Errorf("invalid length %d", length) - } - objName := be.Filename(h) blockBlobClient := be.container.NewBlobClient(objName) - be.sem.GetToken() resp, err := blockBlobClient.DownloadStream(ctx, &blob.DownloadStreamOptions{ Range: azblob.HTTPRange{ Offset: offset, @@ -324,26 +289,20 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, }) if err != nil { - be.sem.ReleaseToken() return nil, err } - return be.sem.ReleaseTokenOnClose(resp.Body, nil), err + return resp.Body, err } // Stat returns information about a blob. func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { - debug.Log("%v", h) - objName := be.Filename(h) blobClient := be.container.NewBlobClient(objName) - be.sem.GetToken() props, err := blobClient.GetProperties(ctx, nil) - be.sem.ReleaseToken() if err != nil { - debug.Log("blob.GetProperties err %v", err) return restic.FileInfo{}, errors.Wrap(err, "blob.GetProperties") } @@ -359,11 +318,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { objName := be.Filename(h) blob := be.container.NewBlobClient(objName) - be.sem.GetToken() _, err := blob.Delete(ctx, &azblob.DeleteBlobOptions{}) - be.sem.ReleaseToken() - - debug.Log("Remove(%v) at %v -> err %v", h, objName, err) if be.IsNotExist(err) { return nil @@ -375,8 +330,6 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - debug.Log("listing %v", t) - prefix, _ := be.Basedir(t) // make sure prefix ends with a slash @@ -393,9 +346,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F lister := be.container.NewListBlobsFlatPager(opts) for lister.More() { - be.sem.GetToken() resp, err := lister.NextPage(ctx) - be.sem.ReleaseToken() if err != nil { return err @@ -433,30 +384,9 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F return ctx.Err() } -// Remove keys for a specified backend type. -func (be *Backend) removeKeys(ctx context.Context, t restic.FileType) error { - return be.List(ctx, t, func(fi restic.FileInfo) error { - return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) - }) -} - // Delete removes all restic keys in the bucket. It will not remove the bucket itself. func (be *Backend) Delete(ctx context.Context) error { - alltypes := []restic.FileType{ - restic.PackFile, - restic.KeyFile, - restic.LockFile, - restic.SnapshotFile, - restic.IndexFile} - - for _, t := range alltypes { - err := be.removeKeys(ctx, t) - if err != nil { - return nil - } - } - - return be.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) + return backend.DefaultDelete(ctx, be) } // Close does nothing diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go index 40dbbf893..738df198d 100644 --- a/internal/backend/b2/b2.go +++ b/internal/backend/b2/b2.go @@ -11,12 +11,10 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" - "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" - "github.com/cenkalti/backoff/v4" "github.com/kurin/blazer/b2" "github.com/kurin/blazer/base" ) @@ -28,7 +26,6 @@ type b2Backend struct { cfg Config listMaxItems int layout.Layout - sem sema.Semaphore canDelete bool } @@ -92,11 +89,6 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend return nil, errors.Wrap(err, "Bucket") } - sem, err := sema.New(cfg.Connections) - if err != nil { - return nil, err - } - be := &b2Backend{ client: client, bucket: bucket, @@ -106,7 +98,6 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend Path: cfg.Prefix, }, listMaxItems: defaultListMaxItems, - sem: sem, canDelete: true, } @@ -134,11 +125,6 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backe return nil, errors.Wrap(err, "NewBucket") } - sem, err := sema.New(cfg.Connections) - if err != nil { - return nil, err - } - be := &b2Backend{ client: client, bucket: bucket, @@ -148,7 +134,6 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backe Path: cfg.Prefix, }, listMaxItems: defaultListMaxItems, - sem: sem, } _, err = be.Stat(ctx, restic.Handle{Type: restic.ConfigFile}) @@ -202,33 +187,18 @@ func (be *b2Backend) IsNotExist(err error) bool { // Load runs fn with a reader that yields the contents of the file at h at the // given offset. func (be *b2Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h)) - if err := h.Valid(); err != nil { - return nil, backoff.Permanent(err) - } - - if offset < 0 { - return nil, errors.New("offset is negative") - } - - if length < 0 { - return nil, errors.Errorf("invalid length %d", length) - } - - ctx, cancel := context.WithCancel(ctx) - - be.sem.GetToken() - name := be.Layout.Filename(h) obj := be.bucket.Object(name) if offset == 0 && length == 0 { - rd := obj.NewReader(ctx) - return be.sem.ReleaseTokenOnClose(rd, cancel), nil + return obj.NewReader(ctx), nil } // pass a negative length to NewRangeReader so that the remainder of the @@ -237,8 +207,7 @@ func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int length = -1 } - rd := obj.NewRangeReader(ctx, offset, int64(length)) - return be.sem.ReleaseTokenOnClose(rd, cancel), nil + return obj.NewRangeReader(ctx, offset, int64(length)), nil } // Save stores data in the backend at the handle. @@ -246,21 +215,12 @@ func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.Rewind ctx, cancel := context.WithCancel(ctx) defer cancel() - if err := h.Valid(); err != nil { - return backoff.Permanent(err) - } - - be.sem.GetToken() - defer be.sem.ReleaseToken() - name := be.Filename(h) - debug.Log("Save %v, name %v", h, name) obj := be.bucket.Object(name) // b2 always requires sha1 checksums for uploaded file parts w := obj.NewWriter(ctx) n, err := io.Copy(w, rd) - debug.Log(" saved %d bytes, err %v", n, err) if err != nil { _ = w.Close() @@ -276,16 +236,10 @@ func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.Rewind // Stat returns information about a blob. func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { - debug.Log("Stat %v", h) - - be.sem.GetToken() - defer be.sem.ReleaseToken() - name := be.Filename(h) obj := be.bucket.Object(name) info, err := obj.Attrs(ctx) if err != nil { - debug.Log("Attrs() err %v", err) return restic.FileInfo{}, errors.Wrap(err, "Stat") } return restic.FileInfo{Size: info.Size, Name: h.Name}, nil @@ -293,11 +247,6 @@ func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileI // Remove removes the blob with the given name and type. func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error { - debug.Log("Remove %v", h) - - be.sem.GetToken() - defer be.sem.ReleaseToken() - // the retry backend will also repeat the remove method up to 10 times for i := 0; i < 3; i++ { obj := be.bucket.Object(be.Filename(h)) @@ -332,22 +281,13 @@ func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error { return errors.New("failed to delete all file versions") } -type semLocker struct { - sema.Semaphore -} - -func (sm *semLocker) Lock() { sm.GetToken() } -func (sm *semLocker) Unlock() { sm.ReleaseToken() } - // List returns a channel that yields all names of blobs of type t. func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - debug.Log("List %v", t) - ctx, cancel := context.WithCancel(ctx) defer cancel() prefix, _ := be.Basedir(t) - iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(be.listMaxItems), b2.ListLocker(&semLocker{be.sem})) + iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(be.listMaxItems)) for iter.Next() { obj := iter.Object() @@ -367,41 +307,14 @@ func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic } } if err := iter.Err(); err != nil { - debug.Log("List: %v", err) return err } return nil } -// Remove keys for a specified backend type. -func (be *b2Backend) removeKeys(ctx context.Context, t restic.FileType) error { - debug.Log("removeKeys %v", t) - return be.List(ctx, t, func(fi restic.FileInfo) error { - return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) - }) -} - // Delete removes all restic keys in the bucket. It will not remove the bucket itself. func (be *b2Backend) Delete(ctx context.Context) error { - alltypes := []restic.FileType{ - restic.PackFile, - restic.KeyFile, - restic.LockFile, - restic.SnapshotFile, - restic.IndexFile} - - for _, t := range alltypes { - err := be.removeKeys(ctx, t) - if err != nil { - return nil - } - } - err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) - if err != nil && be.IsNotExist(err) { - err = nil - } - - return err + return backend.DefaultDelete(ctx, be) } // Close does nothing diff --git a/internal/backend/dryrun/dry_backend.go b/internal/backend/dryrun/dry_backend.go index 37569c320..487e2bc33 100644 --- a/internal/backend/dryrun/dry_backend.go +++ b/internal/backend/dryrun/dry_backend.go @@ -18,10 +18,9 @@ type Backend struct { b restic.Backend } -// statically ensure that RetryBackend implements restic.Backend. +// statically ensure that Backend implements restic.Backend. var _ restic.Backend = &Backend{} -// New returns a new backend that saves all data in a map in memory. func New(be restic.Backend) *Backend { b := &Backend{b: be} debug.Log("created new dry backend") @@ -34,8 +33,6 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe return err } - debug.Log("faked saving %v bytes at %v", rd.Length(), h) - // don't save anything, just return ok return nil } diff --git a/internal/backend/dryrun/dry_backend_test.go b/internal/backend/dryrun/dry_backend_test.go index 6b8f74e0f..69716c340 100644 --- a/internal/backend/dryrun/dry_backend_test.go +++ b/internal/backend/dryrun/dry_backend_test.go @@ -40,11 +40,9 @@ func TestDry(t *testing.T) { {d, "delete", "", "", ""}, {d, "stat", "a", "", "not found"}, {d, "list", "", "", ""}, - {d, "save", "", "", "invalid"}, {m, "save", "a", "baz", ""}, // save a directly to the mem backend {d, "save", "b", "foob", ""}, // b is not saved {d, "save", "b", "xxx", ""}, // no error as b is not saved - {d, "stat", "", "", "invalid"}, {d, "stat", "a", "a 3", ""}, {d, "load", "a", "baz", ""}, {d, "load", "b", "", "not found"}, diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index 77cbcda97..62e5c4954 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -15,7 +15,6 @@ import ( "github.com/pkg/errors" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" - "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" @@ -37,7 +36,6 @@ type Backend struct { gcsClient *storage.Client projectID string connections uint - sem sema.Semaphore bucketName string bucket *storage.BucketHandle prefix string @@ -99,16 +97,10 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { return nil, errors.Wrap(err, "getStorageClient") } - sem, err := sema.New(cfg.Connections) - if err != nil { - return nil, err - } - be := &Backend{ gcsClient: gcsClient, projectID: cfg.ProjectID, connections: cfg.Connections, - sem: sem, bucketName: cfg.Bucket, bucket: gcsClient.Bucket(cfg.Bucket), prefix: cfg.Prefix, @@ -132,14 +124,13 @@ func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) { // // The service account must have the "storage.buckets.create" permission to // create a bucket the does not yet exist. -func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { be, err := open(cfg, rt) if err != nil { return nil, errors.Wrap(err, "open") } // Try to determine if the bucket exists. If it does not, try to create it. - ctx := context.Background() exists, err := be.bucketExists(ctx, be.bucket) if err != nil { if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusForbidden { @@ -169,7 +160,6 @@ func (be *Backend) SetListMaxItems(i int) { // IsNotExist returns true if the error is caused by a not existing file. func (be *Backend) IsNotExist(err error) bool { - debug.Log("IsNotExist(%T, %#v)", err, err) return errors.Is(err, storage.ErrObjectNotExist) } @@ -204,18 +194,8 @@ func (be *Backend) Path() string { // Save stores data in the backend at the handle. func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { - if err := h.Valid(); err != nil { - return err - } - objName := be.Filename(h) - debug.Log("Save %v at %v", h, objName) - - be.sem.GetToken() - - debug.Log("InsertObject(%v, %v)", be.bucketName, objName) - // Set chunk size to zero to disable resumable uploads. // // With a non-zero chunk size (the default is @@ -250,14 +230,10 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe err = cerr } - be.sem.ReleaseToken() - if err != nil { - debug.Log("%v: err %#v: %v", objName, err, err) return errors.Wrap(err, "service.Objects.Insert") } - debug.Log("%v -> %v bytes", objName, wbytes) // sanity check if wbytes != rd.Length() { return errors.Errorf("wrote %d bytes instead of the expected %d bytes", wbytes, rd.Length()) @@ -268,22 +244,13 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe // Load runs fn with a reader that yields the contents of the file at h at the // given offset. func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h)) - if err := h.Valid(); err != nil { - return nil, err - } - - if offset < 0 { - return nil, errors.New("offset is negative") - } - - if length < 0 { - return nil, errors.Errorf("invalid length %d", length) - } if length == 0 { // negative length indicates read till end to GCS lib length = -1 @@ -291,32 +258,21 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, objName := be.Filename(h) - be.sem.GetToken() - - ctx, cancel := context.WithCancel(ctx) - r, err := be.bucket.Object(objName).NewRangeReader(ctx, offset, int64(length)) if err != nil { - cancel() - be.sem.ReleaseToken() return nil, err } - return be.sem.ReleaseTokenOnClose(r, cancel), err + return r, err } // Stat returns information about a blob. func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { - debug.Log("%v", h) - objName := be.Filename(h) - be.sem.GetToken() attr, err := be.bucket.Object(objName).Attrs(ctx) - be.sem.ReleaseToken() if err != nil { - debug.Log("GetObjectAttributes() err %v", err) return restic.FileInfo{}, errors.Wrap(err, "service.Objects.Get") } @@ -327,23 +283,18 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { objName := be.Filename(h) - be.sem.GetToken() err := be.bucket.Object(objName).Delete(ctx) - be.sem.ReleaseToken() - if err == storage.ErrObjectNotExist { + if be.IsNotExist(err) { err = nil } - debug.Log("Remove(%v) at %v -> err %v", h, objName, err) return errors.Wrap(err, "client.RemoveObject") } // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - debug.Log("listing %v", t) - prefix, _ := be.Basedir(t) // make sure prefix ends with a slash @@ -357,9 +308,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F itr := be.bucket.Objects(ctx, &storage.Query{Prefix: prefix}) for { - be.sem.GetToken() attrs, err := itr.Next() - be.sem.ReleaseToken() if err == iterator.Done { break } @@ -389,30 +338,9 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F return ctx.Err() } -// Remove keys for a specified backend type. -func (be *Backend) removeKeys(ctx context.Context, t restic.FileType) error { - return be.List(ctx, t, func(fi restic.FileInfo) error { - return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) - }) -} - // Delete removes all restic keys in the bucket. It will not remove the bucket itself. func (be *Backend) Delete(ctx context.Context) error { - alltypes := []restic.FileType{ - restic.PackFile, - restic.KeyFile, - restic.LockFile, - restic.SnapshotFile, - restic.IndexFile} - - for _, t := range alltypes { - err := be.removeKeys(ctx, t) - if err != nil { - return nil - } - } - - return be.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) + return backend.DefaultDelete(ctx, be) } // Close does nothing. diff --git a/internal/backend/gs/gs_test.go b/internal/backend/gs/gs_test.go index 77f8986f1..19ae8b829 100644 --- a/internal/backend/gs/gs_test.go +++ b/internal/backend/gs/gs_test.go @@ -42,7 +42,7 @@ func newGSTestSuite(t testing.TB) *test.Suite { Create: func(config interface{}) (restic.Backend, error) { cfg := config.(gs.Config) - be, err := gs.Create(cfg, tr) + be, err := gs.Create(context.Background(), cfg, tr) if err != nil { return nil, err } diff --git a/internal/backend/limiter/limiter_backend.go b/internal/backend/limiter/limiter_backend.go index f1b508327..7fcca59cc 100644 --- a/internal/backend/limiter/limiter_backend.go +++ b/internal/backend/limiter/limiter_backend.go @@ -46,6 +46,8 @@ func (r rateLimitedBackend) Load(ctx context.Context, h restic.Handle, length in }) } +func (r rateLimitedBackend) Unwrap() restic.Backend { return r.Backend } + type limitedReader struct { io.Reader writerTo io.WriterTo diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index 1716e0f07..ca806f754 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -10,7 +10,6 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" - "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" @@ -22,7 +21,6 @@ import ( // Local is a backend in a local directory. type Local struct { Config - sem sema.Semaphore layout.Layout backend.Modes } @@ -38,11 +36,6 @@ func open(ctx context.Context, cfg Config) (*Local, error) { return nil, err } - sem, err := sema.New(cfg.Connections) - if err != nil { - return nil, err - } - fi, err := fs.Stat(l.Filename(restic.Handle{Type: restic.ConfigFile})) m := backend.DeriveModesFromFileInfo(fi, err) debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir) @@ -50,7 +43,6 @@ func open(ctx context.Context, cfg Config) (*Local, error) { return &Local{ Config: cfg, Layout: l, - sem: sem, Modes: m, }, nil } @@ -114,11 +106,6 @@ func (b *Local) IsNotExist(err error) bool { // Save stores data in the backend at the handle. func (b *Local) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) (err error) { - debug.Log("Save %v", h) - if err := h.Valid(); err != nil { - return backoff.Permanent(err) - } - finalname := b.Filename(h) dir := filepath.Dir(finalname) @@ -129,9 +116,6 @@ func (b *Local) Save(ctx context.Context, h restic.Handle, rd restic.RewindReade } }() - b.sem.GetToken() - defer b.sem.ReleaseToken() - // Create new file with a temporary name. tmpname := filepath.Base(finalname) + "-tmp-" f, err := tempFile(dir, tmpname) @@ -217,50 +201,28 @@ func (b *Local) Load(ctx context.Context, h restic.Handle, length int, offset in } func (b *Local) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - debug.Log("Load %v, length %v, offset %v", h, length, offset) - if err := h.Valid(); err != nil { - return nil, backoff.Permanent(err) - } - - if offset < 0 { - return nil, errors.New("offset is negative") - } - - b.sem.GetToken() f, err := fs.Open(b.Filename(h)) if err != nil { - b.sem.ReleaseToken() return nil, err } if offset > 0 { _, err = f.Seek(offset, 0) if err != nil { - b.sem.ReleaseToken() _ = f.Close() return nil, err } } - r := b.sem.ReleaseTokenOnClose(f, nil) - if length > 0 { - return backend.LimitReadCloser(r, int64(length)), nil + return backend.LimitReadCloser(f, int64(length)), nil } - return r, nil + return f, nil } // Stat returns information about a blob. func (b *Local) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { - debug.Log("Stat %v", h) - if err := h.Valid(); err != nil { - return restic.FileInfo{}, backoff.Permanent(err) - } - - b.sem.GetToken() - defer b.sem.ReleaseToken() - fi, err := fs.Stat(b.Filename(h)) if err != nil { return restic.FileInfo{}, errors.WithStack(err) @@ -271,12 +233,8 @@ func (b *Local) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, err // Remove removes the blob with the given name and type. func (b *Local) Remove(ctx context.Context, h restic.Handle) error { - debug.Log("Remove %v", h) fn := b.Filename(h) - b.sem.GetToken() - defer b.sem.ReleaseToken() - // reset read-only flag err := fs.Chmod(fn, 0666) if err != nil && !os.IsPermission(err) { @@ -289,8 +247,6 @@ func (b *Local) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) (err error) { - debug.Log("List %v", t) - basedir, subdirs := b.Basedir(t) if subdirs { err = visitDirs(ctx, basedir, fn) @@ -384,13 +340,11 @@ func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error, // Delete removes the repository and all files. func (b *Local) Delete(ctx context.Context) error { - debug.Log("Delete()") return fs.RemoveAll(b.Path) } // Close closes all open files. func (b *Local) Close() error { - debug.Log("Close()") // this does not need to do anything, all open files are closed within the // same function. return nil diff --git a/internal/backend/logger/log.go b/internal/backend/logger/log.go new file mode 100644 index 000000000..6c860cfae --- /dev/null +++ b/internal/backend/logger/log.go @@ -0,0 +1,79 @@ +package logger + +import ( + "context" + "io" + + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/restic" +) + +type Backend struct { + restic.Backend +} + +// statically ensure that Backend implements restic.Backend. +var _ restic.Backend = &Backend{} + +func New(be restic.Backend) *Backend { + return &Backend{Backend: be} +} + +func (be *Backend) IsNotExist(err error) bool { + isNotExist := be.Backend.IsNotExist(err) + debug.Log("IsNotExist(%T, %#v, %v)", err, err, isNotExist) + return isNotExist +} + +// Save adds new Data to the backend. +func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + debug.Log("Save(%v, %v)", h, rd.Length()) + err := be.Backend.Save(ctx, h, rd) + debug.Log(" save err %v", err) + return err +} + +// Remove deletes a file from the backend. +func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { + debug.Log("Remove(%v)", h) + err := be.Backend.Remove(ctx, h) + debug.Log(" remove err %v", err) + return err +} + +func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(io.Reader) error) error { + debug.Log("Load(%v, length %v, offset %v)", h, length, offset) + err := be.Backend.Load(ctx, h, length, offset, fn) + debug.Log(" load err %v", err) + return err +} + +func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { + debug.Log("Stat(%v)", h) + fi, err := be.Backend.Stat(ctx, h) + debug.Log(" stat err %v", err) + return fi, err +} + +func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { + debug.Log("List(%v)", t) + err := be.Backend.List(ctx, t, fn) + debug.Log(" list err %v", err) + return err +} + +func (be *Backend) Delete(ctx context.Context) error { + debug.Log("Delete()") + err := be.Backend.Delete(ctx) + debug.Log(" delete err %v", err) + return err +} + +func (be *Backend) Close() error { + debug.Log("Close()") + err := be.Backend.Close() + debug.Log(" close err %v", err) + return err +} + +func (be *Backend) Unwrap() restic.Backend { return be.Backend } diff --git a/internal/backend/mem/mem_backend.go b/internal/backend/mem/mem_backend.go index 0c46dcd6e..618ef5752 100644 --- a/internal/backend/mem/mem_backend.go +++ b/internal/backend/mem/mem_backend.go @@ -10,12 +10,9 @@ import ( "github.com/cespare/xxhash/v2" "github.com/restic/restic/internal/backend" - "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" - - "github.com/cenkalti/backoff/v4" ) type memMap map[restic.Handle][]byte @@ -32,19 +29,12 @@ const connectionCount = 2 type MemoryBackend struct { data memMap m sync.Mutex - sem sema.Semaphore } // New returns a new backend that saves all data in a map in memory. func New() *MemoryBackend { - sem, err := sema.New(connectionCount) - if err != nil { - panic(err) - } - be := &MemoryBackend{ data: make(memMap), - sem: sem, } debug.Log("created new memory backend") @@ -59,13 +49,6 @@ func (be *MemoryBackend) IsNotExist(err error) bool { // Save adds new Data to the backend. func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { - if err := h.Valid(); err != nil { - return backoff.Permanent(err) - } - - be.sem.GetToken() - defer be.sem.ReleaseToken() - be.m.Lock() defer be.m.Unlock() @@ -102,7 +85,6 @@ func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re } be.data[h] = buf - debug.Log("saved %v bytes at %v", len(buf), h) return ctx.Err() } @@ -114,11 +96,6 @@ func (be *MemoryBackend) Load(ctx context.Context, h restic.Handle, length int, } func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - if err := h.Valid(); err != nil { - return nil, backoff.Permanent(err) - } - - be.sem.GetToken() be.m.Lock() defer be.m.Unlock() @@ -127,21 +104,12 @@ func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length h.Name = "" } - debug.Log("Load %v offset %v len %v", h, offset, length) - - if offset < 0 { - be.sem.ReleaseToken() - return nil, errors.New("offset is negative") - } - if _, ok := be.data[h]; !ok { - be.sem.ReleaseToken() return nil, errNotFound } buf := be.data[h] if offset > int64(len(buf)) { - be.sem.ReleaseToken() return nil, errors.New("offset beyond end of file") } @@ -150,18 +118,11 @@ func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length buf = buf[:length] } - return be.sem.ReleaseTokenOnClose(io.NopCloser(bytes.NewReader(buf)), nil), ctx.Err() + return io.NopCloser(bytes.NewReader(buf)), ctx.Err() } // Stat returns information about a file in the backend. func (be *MemoryBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { - if err := h.Valid(); err != nil { - return restic.FileInfo{}, backoff.Permanent(err) - } - - be.sem.GetToken() - defer be.sem.ReleaseToken() - be.m.Lock() defer be.m.Unlock() @@ -170,8 +131,6 @@ func (be *MemoryBackend) Stat(ctx context.Context, h restic.Handle) (restic.File h.Name = "" } - debug.Log("stat %v", h) - e, ok := be.data[h] if !ok { return restic.FileInfo{}, errNotFound @@ -182,14 +141,9 @@ func (be *MemoryBackend) Stat(ctx context.Context, h restic.Handle) (restic.File // Remove deletes a file from the backend. func (be *MemoryBackend) Remove(ctx context.Context, h restic.Handle) error { - be.sem.GetToken() - defer be.sem.ReleaseToken() - be.m.Lock() defer be.m.Unlock() - debug.Log("Remove %v", h) - h.ContainedBlobType = restic.InvalidBlob if _, ok := be.data[h]; !ok { return errNotFound diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index f4c2897b9..7be5a07c7 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -11,13 +11,11 @@ import ( "path" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" - "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" - - "github.com/cenkalti/backoff/v4" ) // make sure the rest backend implements restic.Backend @@ -27,7 +25,6 @@ var _ restic.Backend = &Backend{} type Backend struct { url *url.URL connections uint - sem sema.Semaphore client http.Client layout.Layout } @@ -40,11 +37,6 @@ const ( // Open opens the REST backend with the given config. func Open(cfg Config, rt http.RoundTripper) (*Backend, error) { - sem, err := sema.New(cfg.Connections) - if err != nil { - return nil, err - } - // use url without trailing slash for layout url := cfg.URL.String() if url[len(url)-1] == '/' { @@ -56,7 +48,6 @@ func Open(cfg Config, rt http.RoundTripper) (*Backend, error) { client: http.Client{Transport: rt}, Layout: &layout.RESTLayout{URL: url, Join: path.Join}, connections: cfg.Connections, - sem: sem, } return be, nil @@ -123,10 +114,6 @@ func (b *Backend) HasAtomicReplace() bool { // Save stores data in the backend at the handle. func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { - if err := h.Valid(); err != nil { - return backoff.Permanent(err) - } - ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -143,9 +130,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea // let's the server know what's coming. req.ContentLength = rd.Length() - b.sem.GetToken() resp, err := b.client.Do(req) - b.sem.ReleaseToken() var cerr error if resp != nil { @@ -212,19 +197,6 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset } func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - debug.Log("Load %v, length %v, offset %v", h, length, offset) - if err := h.Valid(); err != nil { - return nil, backoff.Permanent(err) - } - - if offset < 0 { - return nil, errors.New("offset is negative") - } - - if length < 0 { - return nil, errors.Errorf("invalid length %d", length) - } - req, err := http.NewRequestWithContext(ctx, "GET", b.Filename(h), nil) if err != nil { return nil, errors.WithStack(err) @@ -236,11 +208,8 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o } req.Header.Set("Range", byteRange) req.Header.Set("Accept", ContentTypeV2) - debug.Log("Load(%v) send range %v", h, byteRange) - b.sem.GetToken() resp, err := b.client.Do(req) - b.sem.ReleaseToken() if err != nil { if resp != nil { @@ -265,19 +234,13 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o // Stat returns information about a blob. func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { - if err := h.Valid(); err != nil { - return restic.FileInfo{}, backoff.Permanent(err) - } - req, err := http.NewRequestWithContext(ctx, http.MethodHead, b.Filename(h), nil) if err != nil { return restic.FileInfo{}, errors.WithStack(err) } req.Header.Set("Accept", ContentTypeV2) - b.sem.GetToken() resp, err := b.client.Do(req) - b.sem.ReleaseToken() if err != nil { return restic.FileInfo{}, errors.WithStack(err) } @@ -310,19 +273,13 @@ func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, e // Remove removes the blob with the given name and type. func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { - if err := h.Valid(); err != nil { - return backoff.Permanent(err) - } - req, err := http.NewRequestWithContext(ctx, "DELETE", b.Filename(h), nil) if err != nil { return errors.WithStack(err) } req.Header.Set("Accept", ContentTypeV2) - b.sem.GetToken() resp, err := b.client.Do(req) - b.sem.ReleaseToken() if err != nil { return errors.Wrap(err, "client.Do") @@ -359,9 +316,7 @@ func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi } req.Header.Set("Accept", ContentTypeV2) - b.sem.GetToken() resp, err := b.client.Do(req) - b.sem.ReleaseToken() if err != nil { return errors.Wrap(err, "List") @@ -457,32 +412,7 @@ func (b *Backend) Close() error { return nil } -// Remove keys for a specified backend type. -func (b *Backend) removeKeys(ctx context.Context, t restic.FileType) error { - return b.List(ctx, t, func(fi restic.FileInfo) error { - return b.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) - }) -} - // Delete removes all data in the backend. func (b *Backend) Delete(ctx context.Context) error { - alltypes := []restic.FileType{ - restic.PackFile, - restic.KeyFile, - restic.LockFile, - restic.SnapshotFile, - restic.IndexFile} - - for _, t := range alltypes { - err := b.removeKeys(ctx, t) - if err != nil { - return nil - } - } - - err := b.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) - if err != nil && b.IsNotExist(err) { - return nil - } - return err + return backend.DefaultDelete(ctx, b) } diff --git a/internal/backend/retry/backend_retry.go b/internal/backend/retry/backend_retry.go index b5f2706f4..9c51efedc 100644 --- a/internal/backend/retry/backend_retry.go +++ b/internal/backend/retry/backend_retry.go @@ -191,3 +191,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F return err } + +func (be *Backend) Unwrap() restic.Backend { + return be.Backend +} diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index ad652a206..7b7a761ce 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -13,12 +13,10 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" - "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" - "github.com/cenkalti/backoff/v4" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) @@ -26,7 +24,6 @@ import ( // Backend stores data on an S3 endpoint. type Backend struct { client *minio.Client - sem sema.Semaphore cfg Config layout.Layout } @@ -102,14 +99,8 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro return nil, errors.Wrap(err, "minio.New") } - sem, err := sema.New(cfg.Connections) - if err != nil { - return nil, err - } - be := &Backend{ client: client, - sem: sem, cfg: cfg, } @@ -169,8 +160,6 @@ func isAccessDenied(err error) bool { // IsNotExist returns true if the error is caused by a not existing file. func (be *Backend) IsNotExist(err error) bool { - debug.Log("IsNotExist(%T, %#v)", err, err) - var e minio.ErrorResponse return errors.As(err, &e) && e.Code == "NoSuchKey" } @@ -273,17 +262,8 @@ func (be *Backend) Path() string { // Save stores data in the backend at the handle. func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { - debug.Log("Save %v", h) - - if err := h.Valid(); err != nil { - return backoff.Permanent(err) - } - objName := be.Filename(h) - be.sem.GetToken() - defer be.sem.ReleaseToken() - opts := minio.PutObjectOptions{StorageClass: be.cfg.StorageClass} opts.ContentType = "application/octet-stream" // the only option with the high-level api is to let the library handle the checksum computation @@ -291,11 +271,8 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe // only use multipart uploads for very large files opts.PartSize = 200 * 1024 * 1024 - debug.Log("PutObject(%v, %v, %v)", be.cfg.Bucket, objName, rd.Length()) info, err := be.client.PutObject(ctx, be.cfg.Bucket, objName, io.NopCloser(rd), int64(rd.Length()), opts) - debug.Log("%v -> %v bytes, err %#v: %v", objName, info.Size, err, err) - // sanity check if err == nil && info.Size != rd.Length() { return errors.Errorf("wrote %d bytes instead of the expected %d bytes", info.Size, rd.Length()) @@ -307,32 +284,20 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe // Load runs fn with a reader that yields the contents of the file at h at the // given offset. func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h)) - if err := h.Valid(); err != nil { - return nil, backoff.Permanent(err) - } - - if offset < 0 { - return nil, errors.New("offset is negative") - } - - if length < 0 { - return nil, errors.Errorf("invalid length %d", length) - } - objName := be.Filename(h) opts := minio.GetObjectOptions{} var err error if length > 0 { - debug.Log("range: %v-%v", offset, offset+int64(length)-1) err = opts.SetRange(offset, offset+int64(length)-1) } else if offset > 0 { - debug.Log("range: %v-", offset) err = opts.SetRange(offset, 0) } @@ -340,41 +305,30 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, return nil, errors.Wrap(err, "SetRange") } - be.sem.GetToken() - ctx, cancel := context.WithCancel(ctx) - coreClient := minio.Core{Client: be.client} rd, _, _, err := coreClient.GetObject(ctx, be.cfg.Bucket, objName, opts) if err != nil { - cancel() - be.sem.ReleaseToken() return nil, err } - return be.sem.ReleaseTokenOnClose(rd, cancel), err + return rd, err } // Stat returns information about a blob. func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { - debug.Log("%v", h) - objName := be.Filename(h) var obj *minio.Object opts := minio.GetObjectOptions{} - be.sem.GetToken() obj, err = be.client.GetObject(ctx, be.cfg.Bucket, objName, opts) if err != nil { - debug.Log("GetObject() err %v", err) - be.sem.ReleaseToken() return restic.FileInfo{}, errors.Wrap(err, "client.GetObject") } // make sure that the object is closed properly. defer func() { e := obj.Close() - be.sem.ReleaseToken() if err == nil { err = errors.Wrap(e, "Close") } @@ -382,7 +336,6 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf fi, err := obj.Stat() if err != nil { - debug.Log("Stat() err %v", err) return restic.FileInfo{}, errors.Wrap(err, "Stat") } @@ -393,11 +346,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { objName := be.Filename(h) - be.sem.GetToken() err := be.client.RemoveObject(ctx, be.cfg.Bucket, objName, minio.RemoveObjectOptions{}) - be.sem.ReleaseToken() - - debug.Log("Remove(%v) at %v -> err %v", h, objName, err) if be.IsNotExist(err) { err = nil @@ -409,8 +358,6 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - debug.Log("listing %v", t) - prefix, recursive := be.Basedir(t) // make sure prefix ends with a slash @@ -464,30 +411,9 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F return ctx.Err() } -// Remove keys for a specified backend type. -func (be *Backend) removeKeys(ctx context.Context, t restic.FileType) error { - return be.List(ctx, restic.PackFile, func(fi restic.FileInfo) error { - return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) - }) -} - // Delete removes all restic keys in the bucket. It will not remove the bucket itself. func (be *Backend) Delete(ctx context.Context) error { - alltypes := []restic.FileType{ - restic.PackFile, - restic.KeyFile, - restic.LockFile, - restic.SnapshotFile, - restic.IndexFile} - - for _, t := range alltypes { - err := be.removeKeys(ctx, t) - if err != nil { - return nil - } - } - - return be.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) + return backend.DefaultDelete(ctx, be) } // Close does nothing diff --git a/internal/backend/sema/backend.go b/internal/backend/sema/backend.go new file mode 100644 index 000000000..dd4859ed1 --- /dev/null +++ b/internal/backend/sema/backend.go @@ -0,0 +1,98 @@ +package sema + +import ( + "context" + "io" + + "github.com/cenkalti/backoff/v4" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" +) + +// make sure that connectionLimitedBackend implements restic.Backend +var _ restic.Backend = &connectionLimitedBackend{} + +// connectionLimitedBackend limits the number of concurrent operations. +type connectionLimitedBackend struct { + restic.Backend + sem semaphore +} + +// NewBackend creates a backend that limits the concurrent operations on the underlying backend +func NewBackend(be restic.Backend) restic.Backend { + sem, err := newSemaphore(be.Connections()) + if err != nil { + panic(err) + } + + return &connectionLimitedBackend{ + Backend: be, + sem: sem, + } +} + +// typeDependentLimit acquire a token unless the FileType is a lock file. The returned function +// must be called to release the token. +func (be *connectionLimitedBackend) typeDependentLimit(t restic.FileType) func() { + // allow concurrent lock file operations to ensure that the lock refresh is always possible + if t == restic.LockFile { + return func() {} + } + be.sem.GetToken() + return be.sem.ReleaseToken +} + +// Save adds new Data to the backend. +func (be *connectionLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + if err := h.Valid(); err != nil { + return backoff.Permanent(err) + } + + defer be.typeDependentLimit(h.Type)() + + return be.Backend.Save(ctx, h, rd) +} + +// Load runs fn with a reader that yields the contents of the file at h at the +// given offset. +func (be *connectionLimitedBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + if err := h.Valid(); err != nil { + return backoff.Permanent(err) + } + if offset < 0 { + return backoff.Permanent(errors.New("offset is negative")) + } + if length < 0 { + return backoff.Permanent(errors.Errorf("invalid length %d", length)) + } + + defer be.typeDependentLimit(h.Type)() + + return be.Backend.Load(ctx, h, length, offset, fn) +} + +// Stat returns information about a file in the backend. +func (be *connectionLimitedBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { + if err := h.Valid(); err != nil { + return restic.FileInfo{}, backoff.Permanent(err) + } + + defer be.typeDependentLimit(h.Type)() + + return be.Backend.Stat(ctx, h) +} + +// Remove deletes a file from the backend. +func (be *connectionLimitedBackend) Remove(ctx context.Context, h restic.Handle) error { + if err := h.Valid(); err != nil { + return backoff.Permanent(err) + } + + defer be.typeDependentLimit(h.Type)() + + return be.Backend.Remove(ctx, h) +} + +func (be *connectionLimitedBackend) Unwrap() restic.Backend { + return be.Backend +} diff --git a/internal/backend/sema/backend_test.go b/internal/backend/sema/backend_test.go new file mode 100644 index 000000000..dc599b7f8 --- /dev/null +++ b/internal/backend/sema/backend_test.go @@ -0,0 +1,199 @@ +package sema_test + +import ( + "context" + "io" + "sync/atomic" + "testing" + "time" + + "github.com/restic/restic/internal/backend/mock" + "github.com/restic/restic/internal/backend/sema" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" + "golang.org/x/sync/errgroup" +) + +func TestParameterValidationSave(t *testing.T) { + m := mock.NewBackend() + m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + return nil + } + be := sema.NewBackend(m) + + err := be.Save(context.TODO(), restic.Handle{}, nil) + test.Assert(t, err != nil, "Save() with invalid handle did not return an error") +} + +func TestParameterValidationLoad(t *testing.T) { + m := mock.NewBackend() + m.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + return io.NopCloser(nil), nil + } + + be := sema.NewBackend(m) + nilCb := func(rd io.Reader) error { return nil } + + err := be.Load(context.TODO(), restic.Handle{}, 10, 0, nilCb) + test.Assert(t, err != nil, "Load() with invalid handle did not return an error") + + h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + err = be.Load(context.TODO(), h, 10, -1, nilCb) + test.Assert(t, err != nil, "Save() with negative offset did not return an error") + err = be.Load(context.TODO(), h, -1, 0, nilCb) + test.Assert(t, err != nil, "Save() with negative length did not return an error") +} + +func TestParameterValidationStat(t *testing.T) { + m := mock.NewBackend() + m.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { + return restic.FileInfo{}, nil + } + be := sema.NewBackend(m) + + _, err := be.Stat(context.TODO(), restic.Handle{}) + test.Assert(t, err != nil, "Stat() with invalid handle did not return an error") +} + +func TestParameterValidationRemove(t *testing.T) { + m := mock.NewBackend() + m.RemoveFn = func(ctx context.Context, h restic.Handle) error { + return nil + } + be := sema.NewBackend(m) + + err := be.Remove(context.TODO(), restic.Handle{}) + test.Assert(t, err != nil, "Remove() with invalid handle did not return an error") +} + +func TestUnwrap(t *testing.T) { + m := mock.NewBackend() + be := sema.NewBackend(m) + + unwrapper := be.(restic.BackendUnwrapper) + test.Assert(t, unwrapper.Unwrap() == m, "Unwrap() returned wrong backend") +} + +func countingBlocker() (func(), func(int) int) { + ctr := int64(0) + blocker := make(chan struct{}) + + wait := func() { + // count how many goroutines were allowed by the semaphore + atomic.AddInt64(&ctr, 1) + // block until the test can retrieve the counter + <-blocker + } + + unblock := func(expected int) int { + // give goroutines enough time to block + var blocked int64 + for i := 0; i < 100 && blocked < int64(expected); i++ { + time.Sleep(100 * time.Microsecond) + blocked = atomic.LoadInt64(&ctr) + } + close(blocker) + return int(blocked) + } + + return wait, unblock +} + +func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(be restic.Backend) func() error, unblock func(int) int, isUnlimited bool) { + expectBlocked := int(2) + workerCount := expectBlocked + 1 + + m := mock.NewBackend() + setup(m) + m.ConnectionsFn = func() uint { return uint(expectBlocked) } + be := sema.NewBackend(m) + + var wg errgroup.Group + for i := 0; i < workerCount; i++ { + wg.Go(handler(be)) + } + + if isUnlimited { + expectBlocked = workerCount + } + blocked := unblock(expectBlocked) + test.Assert(t, blocked == expectBlocked, "Unexpected number of goroutines blocked: %v", blocked) + test.OK(t, wg.Wait()) +} + +func TestConcurrencyLimitSave(t *testing.T) { + wait, unblock := countingBlocker() + concurrencyTester(t, func(m *mock.Backend) { + m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + wait() + return nil + } + }, func(be restic.Backend) func() error { + return func() error { + h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + return be.Save(context.TODO(), h, nil) + } + }, unblock, false) +} + +func TestConcurrencyLimitLoad(t *testing.T) { + wait, unblock := countingBlocker() + concurrencyTester(t, func(m *mock.Backend) { + m.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + wait() + return io.NopCloser(nil), nil + } + }, func(be restic.Backend) func() error { + return func() error { + h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + nilCb := func(rd io.Reader) error { return nil } + return be.Load(context.TODO(), h, 10, 0, nilCb) + } + }, unblock, false) +} + +func TestConcurrencyLimitStat(t *testing.T) { + wait, unblock := countingBlocker() + concurrencyTester(t, func(m *mock.Backend) { + m.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { + wait() + return restic.FileInfo{}, nil + } + }, func(be restic.Backend) func() error { + return func() error { + h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + _, err := be.Stat(context.TODO(), h) + return err + } + }, unblock, false) +} + +func TestConcurrencyLimitDelete(t *testing.T) { + wait, unblock := countingBlocker() + concurrencyTester(t, func(m *mock.Backend) { + m.RemoveFn = func(ctx context.Context, h restic.Handle) error { + wait() + return nil + } + }, func(be restic.Backend) func() error { + return func() error { + h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + return be.Remove(context.TODO(), h) + } + }, unblock, false) +} + +func TestConcurrencyUnlimitedLockSave(t *testing.T) { + wait, unblock := countingBlocker() + concurrencyTester(t, func(m *mock.Backend) { + m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + wait() + return nil + } + }, func(be restic.Backend) func() error { + return func() error { + h := restic.Handle{Type: restic.LockFile, Name: "foobar"} + return be.Save(context.TODO(), h, nil) + } + }, unblock, true) +} diff --git a/internal/backend/sema/semaphore.go b/internal/backend/sema/semaphore.go index 7ee912979..c664eef7c 100644 --- a/internal/backend/sema/semaphore.go +++ b/internal/backend/sema/semaphore.go @@ -2,64 +2,30 @@ package sema import ( - "context" - "io" - + "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" ) -// A Semaphore limits access to a restricted resource. -type Semaphore struct { +// A semaphore limits access to a restricted resource. +type semaphore struct { ch chan struct{} } -// New returns a new semaphore with capacity n. -func New(n uint) (Semaphore, error) { +// newSemaphore returns a new semaphore with capacity n. +func newSemaphore(n uint) (semaphore, error) { if n == 0 { - return Semaphore{}, errors.New("capacity must be a positive number") + return semaphore{}, errors.New("capacity must be a positive number") } - return Semaphore{ + return semaphore{ ch: make(chan struct{}, n), }, nil } // GetToken blocks until a Token is available. -func (s Semaphore) GetToken() { s.ch <- struct{}{} } +func (s semaphore) GetToken() { + s.ch <- struct{}{} + debug.Log("acquired token") +} // ReleaseToken returns a token. -func (s Semaphore) ReleaseToken() { <-s.ch } - -// ReleaseTokenOnClose wraps an io.ReadCloser to return a token on Close. -// Before returning the token, cancel, if not nil, will be run -// to free up context resources. -func (s Semaphore) ReleaseTokenOnClose(rc io.ReadCloser, cancel context.CancelFunc) io.ReadCloser { - return &wrapReader{ReadCloser: rc, sem: s, cancel: cancel} -} - -type wrapReader struct { - io.ReadCloser - eofSeen bool - sem Semaphore - cancel context.CancelFunc -} - -func (wr *wrapReader) Read(p []byte) (int, error) { - if wr.eofSeen { // XXX Why do we do this? - return 0, io.EOF - } - - n, err := wr.ReadCloser.Read(p) - if err == io.EOF { - wr.eofSeen = true - } - return n, err -} - -func (wr *wrapReader) Close() error { - err := wr.ReadCloser.Close() - if wr.cancel != nil { - wr.cancel() - } - wr.sem.ReleaseToken() - return err -} +func (s semaphore) ReleaseToken() { <-s.ch } diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index 514dd58da..e97a5f9c8 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -15,7 +15,6 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" - "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -35,7 +34,6 @@ type SFTP struct { posixRename bool - sem sema.Semaphore layout.Layout Config backend.Modes @@ -140,11 +138,7 @@ func Open(ctx context.Context, cfg Config) (*SFTP, error) { } func open(ctx context.Context, sftp *SFTP, cfg Config) (*SFTP, error) { - sem, err := sema.New(cfg.Connections) - if err != nil { - return nil, err - } - + var err error sftp.Layout, err = layout.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path) if err != nil { return nil, err @@ -158,7 +152,6 @@ func open(ctx context.Context, sftp *SFTP, cfg Config) (*SFTP, error) { sftp.Config = cfg sftp.p = cfg.Path - sftp.sem = sem sftp.Modes = m return sftp, nil } @@ -304,22 +297,14 @@ func tempSuffix() string { // Save stores data in the backend at the handle. func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { - debug.Log("Save %v", h) if err := r.clientError(); err != nil { return err } - if err := h.Valid(); err != nil { - return backoff.Permanent(err) - } - filename := r.Filename(h) tmpFilename := filename + "-restic-temp-" + tempSuffix() dirname := r.Dirname(h) - r.sem.GetToken() - defer r.sem.ReleaseToken() - // create new file f, err := r.c.OpenFile(tmpFilename, os.O_CREATE|os.O_EXCL|os.O_WRONLY) @@ -415,77 +400,35 @@ func (r *SFTP) Load(ctx context.Context, h restic.Handle, length int, offset int return backend.DefaultLoad(ctx, h, length, offset, r.openReader, fn) } -// wrapReader wraps an io.ReadCloser to run an additional function on Close. -type wrapReader struct { - io.ReadCloser - io.WriterTo - f func() -} - -func (wr *wrapReader) Close() error { - err := wr.ReadCloser.Close() - wr.f() - return err -} - func (r *SFTP) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - debug.Log("Load %v, length %v, offset %v", h, length, offset) - if err := h.Valid(); err != nil { - return nil, backoff.Permanent(err) - } - - if offset < 0 { - return nil, errors.New("offset is negative") - } - - r.sem.GetToken() f, err := r.c.Open(r.Filename(h)) if err != nil { - r.sem.ReleaseToken() return nil, err } if offset > 0 { _, err = f.Seek(offset, 0) if err != nil { - r.sem.ReleaseToken() _ = f.Close() return nil, err } } - // use custom close wrapper to also provide WriteTo() on the wrapper - rd := &wrapReader{ - ReadCloser: f, - WriterTo: f, - f: func() { - r.sem.ReleaseToken() - }, - } - if length > 0 { // unlimited reads usually use io.Copy which needs WriteTo support at the underlying reader // limited reads are usually combined with io.ReadFull which reads all required bytes into a buffer in one go - return backend.LimitReadCloser(rd, int64(length)), nil + return backend.LimitReadCloser(f, int64(length)), nil } - return rd, nil + return f, nil } // Stat returns information about a blob. func (r *SFTP) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { - debug.Log("Stat(%v)", h) if err := r.clientError(); err != nil { return restic.FileInfo{}, err } - if err := h.Valid(); err != nil { - return restic.FileInfo{}, backoff.Permanent(err) - } - - r.sem.GetToken() - defer r.sem.ReleaseToken() - fi, err := r.c.Lstat(r.Filename(h)) if err != nil { return restic.FileInfo{}, errors.Wrap(err, "Lstat") @@ -496,28 +439,20 @@ func (r *SFTP) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, erro // Remove removes the content stored at name. func (r *SFTP) Remove(ctx context.Context, h restic.Handle) error { - debug.Log("Remove(%v)", h) if err := r.clientError(); err != nil { return err } - r.sem.GetToken() - defer r.sem.ReleaseToken() - return r.c.Remove(r.Filename(h)) } // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. func (r *SFTP) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - debug.Log("List %v", t) - basedir, subdirs := r.Basedir(t) walker := r.c.Walk(basedir) for { - r.sem.GetToken() ok := walker.Step() - r.sem.ReleaseToken() if !ok { break } @@ -572,7 +507,6 @@ var closeTimeout = 2 * time.Second // Close closes the sftp connection and terminates the underlying command. func (r *SFTP) Close() error { - debug.Log("Close") if r == nil { return nil } diff --git a/internal/backend/smb/smb.go b/internal/backend/smb/smb.go index 65e22cc0c..9a71df417 100644 --- a/internal/backend/smb/smb.go +++ b/internal/backend/smb/smb.go @@ -17,7 +17,6 @@ import ( "github.com/hirochachacha/go-smb2" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" - "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -46,7 +45,6 @@ import ( // Backend stores data on an SMB endpoint. type Backend struct { - sem sema.Semaphore Config layout.Layout backend.Modes @@ -71,14 +69,8 @@ func open(ctx context.Context, cfg Config) (*Backend, error) { return nil, err } - sem, err := sema.New(cfg.Connections) - if err != nil { - return nil, err - } - b := &Backend{ Config: cfg, - sem: sem, Layout: l, } @@ -174,11 +166,6 @@ func (b *Backend) Join(p ...string) string { // Save stores data in the backend at the handle. func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) (err error) { - debug.Log("Save %v", h) - if err := h.Valid(); err != nil { - return backoff.Permanent(err) - } - filename := b.Filename(h) tmpFilename := filename + "-restic-temp-" + tempSuffix() dir := filepath.Dir(tmpFilename) @@ -190,9 +177,6 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea } }() - b.sem.GetToken() - defer b.sem.ReleaseToken() - b.addSession() // Show session in use defer b.removeSession() @@ -282,15 +266,6 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset } func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - debug.Log("Load %v, length %v, offset %v", h, length, offset) - if err := h.Valid(); err != nil { - return nil, backoff.Permanent(err) - } - - if offset < 0 { - return nil, errors.New("offset is negative") - } - b.addSession() // Show session in use defer b.removeSession() cn, err := b.getConnection(ctx, b.ShareName) @@ -299,41 +274,28 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o } defer b.putConnection(cn) - b.sem.GetToken() f, err := cn.smbShare.Open(b.Filename(h)) if err != nil { - b.sem.ReleaseToken() return nil, err } if offset > 0 { _, err = f.Seek(offset, 0) if err != nil { - b.sem.ReleaseToken() _ = f.Close() return nil, err } } - r := b.sem.ReleaseTokenOnClose(f, nil) - if length > 0 { - return backend.LimitReadCloser(r, int64(length)), nil + return backend.LimitReadCloser(f, int64(length)), nil } - return r, nil + return f, nil } // Stat returns information about a blob. func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { - debug.Log("Stat %v", h) - if err := h.Valid(); err != nil { - return restic.FileInfo{}, backoff.Permanent(err) - } - - b.sem.GetToken() - defer b.sem.ReleaseToken() - cn, err := b.getConnection(ctx, b.ShareName) if err != nil { return restic.FileInfo{}, err @@ -350,12 +312,8 @@ func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, e // Remove removes the blob with the given name and type. func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { - debug.Log("Remove %v", h) fn := b.Filename(h) - b.sem.GetToken() - defer b.sem.ReleaseToken() - cn, err := b.getConnection(ctx, b.ShareName) if err != nil { return err @@ -374,8 +332,6 @@ func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) (err error) { - debug.Log("List %v", t) - cn, err := b.getConnection(ctx, b.ShareName) if err != nil { return err @@ -475,7 +431,6 @@ func (b *Backend) visitFiles(ctx context.Context, cn *conn, dir string, fn func( // Delete removes the repository and all files. func (b *Backend) Delete(ctx context.Context) error { - debug.Log("Delete()") cn, err := b.getConnection(ctx, b.ShareName) if err != nil { return err @@ -486,7 +441,6 @@ func (b *Backend) Delete(ctx context.Context) error { // Close closes all open files. func (b *Backend) Close() error { - debug.Log("Close()") err := b.drainPool() return err } diff --git a/internal/backend/swift/swift.go b/internal/backend/swift/swift.go index 764c7bb62..cfa9ed665 100644 --- a/internal/backend/swift/swift.go +++ b/internal/backend/swift/swift.go @@ -15,12 +15,10 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" - "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" - "github.com/cenkalti/backoff/v4" "github.com/ncw/swift/v2" ) @@ -28,7 +26,6 @@ import ( type beSwift struct { conn *swift.Connection connections uint - sem sema.Semaphore container string // Container name prefix string // Prefix of object names in the container layout.Layout @@ -42,11 +39,6 @@ var _ restic.Backend = &beSwift{} func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { debug.Log("config %#v", cfg) - sem, err := sema.New(cfg.Connections) - if err != nil { - return nil, err - } - be := &beSwift{ conn: &swift.Connection{ UserName: cfg.UserName, @@ -72,7 +64,6 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend Transport: rt, }, connections: cfg.Connections, - sem: sem, container: cfg.Container, prefix: cfg.Prefix, Layout: &layout.DefaultLayout{ @@ -143,18 +134,6 @@ func (be *beSwift) Load(ctx context.Context, h restic.Handle, length int, offset } func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - debug.Log("Load %v, length %v, offset %v", h, length, offset) - if err := h.Valid(); err != nil { - return nil, backoff.Permanent(err) - } - - if offset < 0 { - return nil, errors.New("offset is negative") - } - - if length < 0 { - return nil, errors.Errorf("invalid length %d", length) - } objName := be.Filename(h) @@ -167,59 +146,34 @@ func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int, headers["Range"] = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1) } - if _, ok := headers["Range"]; ok { - debug.Log("Load(%v) send range %v", h, headers["Range"]) - } - - be.sem.GetToken() obj, _, err := be.conn.ObjectOpen(ctx, be.container, objName, false, headers) if err != nil { - debug.Log(" err %v", err) - be.sem.ReleaseToken() return nil, errors.Wrap(err, "conn.ObjectOpen") } - return be.sem.ReleaseTokenOnClose(obj, nil), nil + return obj, nil } // Save stores data in the backend at the handle. func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { - if err := h.Valid(); err != nil { - return backoff.Permanent(err) - } - objName := be.Filename(h) - - debug.Log("Save %v at %v", h, objName) - - be.sem.GetToken() - defer be.sem.ReleaseToken() - encoding := "binary/octet-stream" - debug.Log("PutObject(%v, %v, %v)", be.container, objName, encoding) hdr := swift.Headers{"Content-Length": strconv.FormatInt(rd.Length(), 10)} _, err := be.conn.ObjectPut(ctx, be.container, objName, rd, true, hex.EncodeToString(rd.Hash()), encoding, hdr) // swift does not return the upload length - debug.Log("%v, err %#v", objName, err) return errors.Wrap(err, "client.PutObject") } // Stat returns information about a blob. func (be *beSwift) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { - debug.Log("%v", h) - objName := be.Filename(h) - be.sem.GetToken() - defer be.sem.ReleaseToken() - obj, _, err := be.conn.Object(ctx, be.container, objName) if err != nil { - debug.Log("Object() err %v", err) return restic.FileInfo{}, errors.Wrap(err, "conn.Object") } @@ -230,27 +184,19 @@ func (be *beSwift) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error { objName := be.Filename(h) - be.sem.GetToken() - defer be.sem.ReleaseToken() - err := be.conn.ObjectDelete(ctx, be.container, objName) - debug.Log("Remove(%v) -> err %v", h, err) return errors.Wrap(err, "conn.ObjectDelete") } // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. func (be *beSwift) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - debug.Log("listing %v", t) - prefix, _ := be.Basedir(t) prefix += "/" err := be.conn.ObjectsWalk(ctx, be.container, &swift.ObjectsOpts{Prefix: prefix}, func(ctx context.Context, opts *swift.ObjectsOpts) (interface{}, error) { - be.sem.GetToken() newObjects, err := be.conn.Objects(ctx, be.container, opts) - be.sem.ReleaseToken() if err != nil { return nil, errors.Wrap(err, "conn.ObjectNames") @@ -285,13 +231,6 @@ func (be *beSwift) List(ctx context.Context, t restic.FileType, fn func(restic.F return ctx.Err() } -// Remove keys for a specified backend type. -func (be *beSwift) removeKeys(ctx context.Context, t restic.FileType) error { - return be.List(ctx, t, func(fi restic.FileInfo) error { - return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) - }) -} - // IsNotExist returns true if the error is caused by a not existing file. func (be *beSwift) IsNotExist(err error) bool { var e *swift.Error @@ -301,26 +240,7 @@ func (be *beSwift) IsNotExist(err error) bool { // Delete removes all restic objects in the container. // It will not remove the container itself. func (be *beSwift) Delete(ctx context.Context) error { - alltypes := []restic.FileType{ - restic.PackFile, - restic.KeyFile, - restic.LockFile, - restic.SnapshotFile, - restic.IndexFile} - - for _, t := range alltypes { - err := be.removeKeys(ctx, t) - if err != nil { - return nil - } - } - - err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) - if err != nil && !be.IsNotExist(err) { - return err - } - - return nil + return backend.DefaultDelete(ctx, be) } // Close does nothing diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go index b98af59c3..53a10f446 100644 --- a/internal/backend/test/tests.go +++ b/internal/backend/test/tests.go @@ -124,17 +124,7 @@ func (s *Suite) TestLoad(t *testing.T) { b := s.open(t) defer s.close(t, b) - noop := func(rd io.Reader) error { - return nil - } - - err := b.Load(context.TODO(), restic.Handle{}, 0, 0, noop) - if err == nil { - t.Fatalf("Load() did not return an error for invalid handle") - } - test.Assert(t, !b.IsNotExist(err), "IsNotExist() should not accept an invalid handle error: %v", err) - - err = testLoad(b, restic.Handle{Type: restic.PackFile, Name: "foobar"}, 0, 0) + err := testLoad(b, restic.Handle{Type: restic.PackFile, Name: "foobar"}, 0, 0) if err == nil { t.Fatalf("Load() did not return an error for non-existing blob") } @@ -153,11 +143,6 @@ func (s *Suite) TestLoad(t *testing.T) { t.Logf("saved %d bytes as %v", length, handle) - err = b.Load(context.TODO(), handle, 100, -1, noop) - if err == nil { - t.Fatalf("Load() returned no error for negative offset!") - } - err = b.Load(context.TODO(), handle, 0, 0, func(rd io.Reader) error { _, err := io.Copy(io.Discard, rd) if err != nil { diff --git a/internal/backend/utils.go b/internal/backend/utils.go index d2ac44670..cd6614f34 100644 --- a/internal/backend/utils.go +++ b/internal/backend/utils.go @@ -62,6 +62,7 @@ func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser { func DefaultLoad(ctx context.Context, h restic.Handle, length int, offset int64, openReader func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error), fn func(rd io.Reader) error) error { + rd, err := openReader(ctx, h, length, offset) if err != nil { return err @@ -74,6 +75,31 @@ func DefaultLoad(ctx context.Context, h restic.Handle, length int, offset int64, return rd.Close() } +// DefaultDelete removes all restic keys in the bucket. It will not remove the bucket itself. +func DefaultDelete(ctx context.Context, be restic.Backend) error { + alltypes := []restic.FileType{ + restic.PackFile, + restic.KeyFile, + restic.LockFile, + restic.SnapshotFile, + restic.IndexFile} + + for _, t := range alltypes { + err := be.List(ctx, t, func(fi restic.FileInfo) error { + return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) + }) + if err != nil { + return nil + } + } + err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) + if err != nil && be.IsNotExist(err) { + err = nil + } + + return err +} + type memorizedLister struct { fileInfos []restic.FileInfo tpe restic.FileType diff --git a/internal/cache/backend.go b/internal/cache/backend.go index a707f8243..7c9df0f8e 100644 --- a/internal/cache/backend.go +++ b/internal/cache/backend.go @@ -83,7 +83,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea if err != nil { debug.Log("unable to save %v to cache: %v", h, err) _ = b.Cache.remove(h) - return nil + return err } return nil @@ -106,11 +106,19 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error { return nil } + defer func() { + // signal other waiting goroutines that the file may now be cached + close(finish) + + // remove the finish channel from the map + b.inProgressMutex.Lock() + delete(b.inProgress, h) + b.inProgressMutex.Unlock() + }() + // test again, maybe the file was cached in the meantime if !b.Cache.Has(h) { - // nope, it's still not in the cache, pull it from the repo and save it - err := b.Backend.Load(ctx, h, 0, 0, func(rd io.Reader) error { return b.Cache.Save(h, rd) }) @@ -118,16 +126,9 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error { // try to remove from the cache, ignore errors _ = b.Cache.remove(h) } + return err } - // signal other waiting goroutines that the file may now be cached - close(finish) - - // remove the finish channel from the map - b.inProgressMutex.Lock() - delete(b.inProgress, h) - b.inProgressMutex.Unlock() - return nil } @@ -178,11 +179,13 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset debug.Log("auto-store %v in the cache", h) err = b.cacheFile(ctx, h) - if err == nil { - inCache, err = b.loadFromCache(ctx, h, length, offset, consumer) - if inCache { - return err - } + if err != nil { + return err + } + + inCache, err = b.loadFromCache(ctx, h, length, offset, consumer) + if inCache { + return err } debug.Log("error caching %v: %v, falling back to backend", h, err) @@ -211,3 +214,7 @@ func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, e func (b *Backend) IsNotExist(err error) bool { return b.Backend.IsNotExist(err) } + +func (b *Backend) Unwrap() restic.Backend { + return b.Backend +} diff --git a/internal/cache/file_test.go b/internal/cache/file_test.go index 111a2430f..335e78aba 100644 --- a/internal/cache/file_test.go +++ b/internal/cache/file_test.go @@ -11,8 +11,10 @@ import ( "time" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" + rtest "github.com/restic/restic/internal/test" "golang.org/x/sync/errgroup" ) @@ -266,3 +268,19 @@ func TestFileSaveConcurrent(t *testing.T) { saved := load(t, c, h) test.Equals(t, data, saved) } + +func TestFileSaveAfterDamage(t *testing.T) { + c := TestNewCache(t) + rtest.OK(t, fs.RemoveAll(c.path)) + + // save a few bytes of data in the cache + data := test.Random(123456789, 42) + id := restic.Hash(data) + h := restic.Handle{ + Type: restic.PackFile, + Name: id.String(), + } + if err := c.Save(h, bytes.NewReader(data)); err == nil { + t.Fatal("Missing error when saving to deleted cache directory") + } +} diff --git a/internal/checker/checker_test.go b/internal/checker/checker_test.go index 775484652..6405ecfbd 100644 --- a/internal/checker/checker_test.go +++ b/internal/checker/checker_test.go @@ -331,10 +331,6 @@ func (erd errorReadCloser) Read(p []byte) (int, error) { // induceError flips a bit in the slice. func induceError(data []byte) { - if rand.Float32() < 0.2 { - return - } - pos := rand.Intn(len(data)) data[pos] ^= 1 } diff --git a/internal/debug/stacktrace.go b/internal/debug/stacktrace.go new file mode 100644 index 000000000..a8db83160 --- /dev/null +++ b/internal/debug/stacktrace.go @@ -0,0 +1,15 @@ +package debug + +import "runtime" + +func DumpStacktrace() string { + buf := make([]byte, 128*1024) + + for { + l := runtime.Stack(buf, true) + if l < len(buf) { + return string(buf[:l]) + } + buf = make([]byte, len(buf)*2) + } +} diff --git a/internal/debug/testing.go b/internal/debug/testing.go new file mode 100644 index 000000000..c9ceae0ea --- /dev/null +++ b/internal/debug/testing.go @@ -0,0 +1,23 @@ +package debug + +import ( + "log" + "os" + "testing" +) + +// TestLogToStderr configures debug to log to stderr if not the debug log is +// not already configured and returns whether logging was enabled. +func TestLogToStderr(t testing.TB) bool { + if opts.isEnabled { + return false + } + opts.logger = log.New(os.Stderr, "", log.LstdFlags) + opts.isEnabled = true + return true +} + +func TestDisableLog(t testing.TB) { + opts.logger = nil + opts.isEnabled = false +} diff --git a/internal/dump/tar.go b/internal/dump/tar.go index 6e87aabe5..df9ea429d 100644 --- a/internal/dump/tar.go +++ b/internal/dump/tar.go @@ -3,6 +3,7 @@ package dump import ( "archive/tar" "context" + "fmt" "os" "path/filepath" "strings" @@ -94,9 +95,8 @@ func (d *Dumper) dumpNodeTar(ctx context.Context, node *restic.Node, w *tar.Writ err = w.WriteHeader(header) if err != nil { - return errors.Wrap(err, "TarHeader") + return fmt.Errorf("writing header for %q: %w", node.Path, err) } - return d.writeNode(ctx, w, node) } diff --git a/internal/dump/tar_test.go b/internal/dump/tar_test.go index 0f2cb27a8..3556e6aeb 100644 --- a/internal/dump/tar_test.go +++ b/internal/dump/tar_test.go @@ -3,6 +3,8 @@ package dump import ( "archive/tar" "bytes" + "context" + "errors" "fmt" "io" "os" @@ -12,6 +14,8 @@ import ( "time" "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" ) func TestWriteTar(t *testing.T) { @@ -112,3 +116,29 @@ func checkTar(t *testing.T, testDir string, srcTar *bytes.Buffer) error { return nil } + +// #4307. +func TestFieldTooLong(t *testing.T) { + const maxSpecialFileSize = 1 << 20 // Unexported limit in archive/tar. + + node := restic.Node{ + Name: "file_with_xattr", + Path: "/file_with_xattr", + Type: "file", + Mode: 0644, + ExtendedAttributes: []restic.ExtendedAttribute{ + { + Name: "user.way_too_large", + Value: make([]byte, 2*maxSpecialFileSize), + }, + }, + } + + d := Dumper{format: "tar"} + err := d.dumpNodeTar(context.Background(), &node, tar.NewWriter(io.Discard)) + + // We want a tar.ErrFieldTooLong that has the filename. + rtest.Assert(t, errors.Is(err, tar.ErrFieldTooLong), "wrong type %T", err) + rtest.Assert(t, strings.Contains(err.Error(), node.Path), + "no filename in %q", err) +} diff --git a/internal/fuse/file.go b/internal/fuse/file.go index 28ff5d450..35bc2a73e 100644 --- a/internal/fuse/file.go +++ b/internal/fuse/file.go @@ -50,7 +50,7 @@ func (f *file) Attr(ctx context.Context, a *fuse.Attr) error { a.Inode = f.inode a.Mode = f.node.Mode a.Size = f.node.Size - a.Blocks = (f.node.Size / blockSize) + 1 + a.Blocks = (f.node.Size + blockSize - 1) / blockSize a.BlockSize = blockSize a.Nlink = uint32(f.node.Links) diff --git a/internal/fuse/fuse_test.go b/internal/fuse/fuse_test.go index e71bf6fee..9ca1ec0c6 100644 --- a/internal/fuse/fuse_test.go +++ b/internal/fuse/fuse_test.go @@ -8,6 +8,7 @@ import ( "context" "math/rand" "os" + "strings" "testing" "time" @@ -216,6 +217,37 @@ func testTopUIDGID(t *testing.T, cfg Config, repo restic.Repository, uid, gid ui rtest.Equals(t, uint32(0), attr.Gid) } +// Test reporting of fuse.Attr.Blocks in multiples of 512. +func TestBlocks(t *testing.T) { + root := &Root{} + + for _, c := range []struct { + size, blocks uint64 + }{ + {0, 0}, + {1, 1}, + {511, 1}, + {512, 1}, + {513, 2}, + {1024, 2}, + {1025, 3}, + {41253, 81}, + } { + target := strings.Repeat("x", int(c.size)) + + for _, n := range []fs.Node{ + &file{root: root, node: &restic.Node{Size: uint64(c.size)}}, + &link{root: root, node: &restic.Node{LinkTarget: target}}, + &snapshotLink{root: root, snapshot: &restic.Snapshot{}, target: target}, + } { + var a fuse.Attr + err := n.Attr(context.TODO(), &a) + rtest.OK(t, err) + rtest.Equals(t, c.blocks, a.Blocks) + } + } +} + func TestInodeFromNode(t *testing.T) { node := &restic.Node{Name: "foo.txt", Type: "chardev", Links: 2} ino1 := inodeFromNode(1, node) @@ -226,6 +258,17 @@ func TestInodeFromNode(t *testing.T) { ino1 = inodeFromNode(1, node) ino2 = inodeFromNode(2, node) rtest.Assert(t, ino1 != ino2, "same inode %d but different parent", ino1) + + // Regression test: in a path a/b/b, the grandchild should not get the + // same inode as the grandparent. + a := &restic.Node{Name: "a", Type: "dir", Links: 2} + ab := &restic.Node{Name: "b", Type: "dir", Links: 2} + abb := &restic.Node{Name: "b", Type: "dir", Links: 2} + inoA := inodeFromNode(1, a) + inoAb := inodeFromNode(inoA, ab) + inoAbb := inodeFromNode(inoAb, abb) + rtest.Assert(t, inoA != inoAb, "inode(a/b) = inode(a)") + rtest.Assert(t, inoA != inoAbb, "inode(a/b/b) = inode(a)") } var sink uint64 diff --git a/internal/fuse/inode.go b/internal/fuse/inode.go index de975b167..5e2ece4ac 100644 --- a/internal/fuse/inode.go +++ b/internal/fuse/inode.go @@ -10,9 +10,11 @@ import ( "github.com/restic/restic/internal/restic" ) +const prime = 11400714785074694791 // prime1 from xxhash. + // inodeFromName generates an inode number for a file in a meta dir. func inodeFromName(parent uint64, name string) uint64 { - inode := parent ^ xxhash.Sum64String(cleanupNodeName(name)) + inode := prime*parent ^ xxhash.Sum64String(cleanupNodeName(name)) // Inode 0 is invalid and 1 is the root. Remap those. if inode < 2 { @@ -33,7 +35,7 @@ func inodeFromNode(parent uint64, node *restic.Node) (inode uint64) { } else { // Else, use the name and the parent inode. // node.{DeviceID,Inode} may not even be reliable. - inode = parent ^ xxhash.Sum64String(cleanupNodeName(node.Name)) + inode = prime*parent ^ xxhash.Sum64String(cleanupNodeName(node.Name)) } // Inode 0 is invalid and 1 is the root. Remap those. diff --git a/internal/fuse/link.go b/internal/fuse/link.go index f910aadc4..47ee666a3 100644 --- a/internal/fuse/link.go +++ b/internal/fuse/link.go @@ -42,7 +42,7 @@ func (l *link) Attr(ctx context.Context, a *fuse.Attr) error { a.Nlink = uint32(l.node.Links) a.Size = uint64(len(l.node.LinkTarget)) - a.Blocks = 1 + a.Size/blockSize + a.Blocks = (a.Size + blockSize - 1) / blockSize return nil } diff --git a/internal/fuse/snapshots_dir.go b/internal/fuse/snapshots_dir.go index 977d0ab17..c19155741 100644 --- a/internal/fuse/snapshots_dir.go +++ b/internal/fuse/snapshots_dir.go @@ -142,7 +142,7 @@ func (l *snapshotLink) Attr(ctx context.Context, a *fuse.Attr) error { a.Inode = l.inode a.Mode = os.ModeSymlink | 0777 a.Size = uint64(len(l.target)) - a.Blocks = 1 + a.Size/blockSize + a.Blocks = (a.Size + blockSize - 1) / blockSize a.Uid = l.root.uid a.Gid = l.root.gid a.Atime = l.snapshot.Time diff --git a/internal/migrations/s3_layout.go b/internal/migrations/s3_layout.go index d42b94bf8..a5293ef16 100644 --- a/internal/migrations/s3_layout.go +++ b/internal/migrations/s3_layout.go @@ -8,7 +8,6 @@ import ( "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/s3" - "github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -22,24 +21,26 @@ func init() { // "default" layout. type S3Layout struct{} -func toS3Backend(repo restic.Repository) *s3.Backend { - b := repo.Backend() - // unwrap cache - if be, ok := b.(*cache.Backend); ok { - b = be.Backend - } +func toS3Backend(b restic.Backend) *s3.Backend { + for b != nil { + if be, ok := b.(*s3.Backend); ok { + return be + } - be, ok := b.(*s3.Backend) - if !ok { - debug.Log("backend is not s3") - return nil + if be, ok := b.(restic.BackendUnwrapper); ok { + b = be.Unwrap() + } else { + // not the backend we're looking for + break + } } - return be + debug.Log("backend is not s3") + return nil } // Check tests whether the migration can be applied. func (m *S3Layout) Check(ctx context.Context, repo restic.Repository) (bool, string, error) { - be := toS3Backend(repo) + be := toS3Backend(repo.Backend()) if be == nil { debug.Log("backend is not s3") return false, "backend is not s3", nil @@ -91,7 +92,7 @@ func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l layout.Layou // Apply runs the migration. func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error { - be := toS3Backend(repo) + be := toS3Backend(repo.Backend()) if be == nil { debug.Log("backend is not s3") return errors.New("backend is not s3") diff --git a/internal/migrations/s3_layout_test.go b/internal/migrations/s3_layout_test.go new file mode 100644 index 000000000..ad0eedea6 --- /dev/null +++ b/internal/migrations/s3_layout_test.go @@ -0,0 +1,27 @@ +package migrations + +import ( + "testing" + + "github.com/restic/restic/internal/backend/mock" + "github.com/restic/restic/internal/backend/s3" + "github.com/restic/restic/internal/cache" + "github.com/restic/restic/internal/test" +) + +func TestS3UnwrapBackend(t *testing.T) { + // toS3Backend(b restic.Backend) *s3.Backend + + m := mock.NewBackend() + test.Assert(t, toS3Backend(m) == nil, "mock backend is not an s3 backend") + + // uninitialized fake backend for testing + s3 := &s3.Backend{} + test.Assert(t, toS3Backend(s3) == s3, "s3 was not returned") + + c := &cache.Backend{Backend: s3} + test.Assert(t, toS3Backend(c) == s3, "failed to unwrap s3 backend") + + c.Backend = m + test.Assert(t, toS3Backend(c) == nil, "a wrapped mock backend is not an s3 backend") +} diff --git a/internal/restic/backend.go b/internal/restic/backend.go index bc139fc8b..b01071132 100644 --- a/internal/restic/backend.go +++ b/internal/restic/backend.go @@ -70,6 +70,11 @@ type Backend interface { Delete(ctx context.Context) error } +type BackendUnwrapper interface { + // Unwrap returns the underlying backend or nil if there is none. + Unwrap() Backend +} + // FileInfo is contains information about a file in the backend. type FileInfo struct { Size int64 diff --git a/internal/restic/parallel.go b/internal/restic/parallel.go index df160f018..34a2a019c 100644 --- a/internal/restic/parallel.go +++ b/internal/restic/parallel.go @@ -41,7 +41,7 @@ func ParallelList(ctx context.Context, r Lister, t FileType, parallelism uint, f // a worker receives an index ID from ch, loads the index, and sends it to indexCh worker := func() error { for fi := range ch { - debug.Log("worker got file %v", fi.ID.Str()) + debug.Log("worker got file %v/%v", t, fi.ID.Str()) err := fn(ctx, fi.ID, fi.Size) if err != nil { return err diff --git a/internal/restic/snapshot.go b/internal/restic/snapshot.go index 58d863526..1f6e4534b 100644 --- a/internal/restic/snapshot.go +++ b/internal/restic/snapshot.go @@ -61,7 +61,7 @@ func LoadSnapshot(ctx context.Context, loader LoaderUnpacked, id ID) (*Snapshot, sn := &Snapshot{id: &id} err := LoadJSONUnpacked(ctx, loader, SnapshotFile, id, sn) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to load snapshot %v: %w", id.Str(), err) } return sn, nil diff --git a/internal/restic/snapshot_find.go b/internal/restic/snapshot_find.go index 4d4bb4957..8d6f8c4b1 100644 --- a/internal/restic/snapshot_find.go +++ b/internal/restic/snapshot_find.go @@ -46,6 +46,7 @@ func (f *SnapshotFilter) findLatest(ctx context.Context, be Lister, loader Loade } absTargets = append(absTargets, filepath.Clean(target)) } + f.Paths = absTargets var latest *Snapshot diff --git a/internal/restic/snapshot_group.go b/internal/restic/snapshot_group.go index c3f3307f6..9efae2ff6 100644 --- a/internal/restic/snapshot_group.go +++ b/internal/restic/snapshot_group.go @@ -68,7 +68,7 @@ type SnapshotGroupKey struct { } // GroupSnapshots takes a list of snapshots and a grouping criteria and creates -// a group list of snapshots. +// a grouped list of snapshots. func GroupSnapshots(snapshots Snapshots, groupBy SnapshotGroupByOptions) (map[string]Snapshots, bool, error) { // group by hostname and dirs snapshotGroups := make(map[string]Snapshots) diff --git a/internal/restic/snapshot_policy.go b/internal/restic/snapshot_policy.go index 3271140aa..228e4c88a 100644 --- a/internal/restic/snapshot_policy.go +++ b/internal/restic/snapshot_policy.go @@ -31,23 +31,22 @@ func (e ExpirePolicy) String() (s string) { var keeps []string var keepw []string - if e.Last > 0 { - keeps = append(keeps, fmt.Sprintf("%d latest", e.Last)) - } - if e.Hourly > 0 { - keeps = append(keeps, fmt.Sprintf("%d hourly", e.Hourly)) - } - if e.Daily > 0 { - keeps = append(keeps, fmt.Sprintf("%d daily", e.Daily)) - } - if e.Weekly > 0 { - keeps = append(keeps, fmt.Sprintf("%d weekly", e.Weekly)) - } - if e.Monthly > 0 { - keeps = append(keeps, fmt.Sprintf("%d monthly", e.Monthly)) - } - if e.Yearly > 0 { - keeps = append(keeps, fmt.Sprintf("%d yearly", e.Yearly)) + for _, opt := range []struct { + count int + descr string + }{ + {e.Last, "latest"}, + {e.Hourly, "hourly"}, + {e.Daily, "daily"}, + {e.Weekly, "weekly"}, + {e.Monthly, "monthly"}, + {e.Yearly, "yearly"}, + } { + if opt.count > 0 { + keeps = append(keeps, fmt.Sprintf("%d %s", opt.count, opt.descr)) + } else if opt.count == -1 { + keeps = append(keeps, fmt.Sprintf("all %s", opt.descr)) + } } if !e.WithinHourly.Zero() { @@ -100,13 +99,7 @@ func (e ExpirePolicy) String() (s string) { return s } -// Sum returns the maximum number of snapshots to be kept according to this -// policy. -func (e ExpirePolicy) Sum() int { - return e.Last + e.Hourly + e.Daily + e.Weekly + e.Monthly + e.Yearly -} - -// Empty returns true iff no policy has been configured (all values zero). +// Empty returns true if no policy has been configured (all values zero). func (e ExpirePolicy) Empty() bool { if len(e.Tags) != 0 { return false @@ -260,13 +253,16 @@ func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots, reason // Now update the other buckets and see if they have some counts left. for i, b := range buckets { - if b.Count > 0 { + // -1 means "keep all" + if b.Count > 0 || b.Count == -1 { val := b.bucker(cur.Time, nr) if val != b.Last { debug.Log("keep %v %v, bucker %v, val %v\n", cur.Time, cur.id.Str(), i, val) keepSnap = true buckets[i].Last = val - buckets[i].Count-- + if buckets[i].Count > 0 { + buckets[i].Count-- + } keepSnapReasons = append(keepSnapReasons, b.reason) } } diff --git a/internal/restic/snapshot_policy_test.go b/internal/restic/snapshot_policy_test.go index 918ea4ec7..75f0f18f4 100644 --- a/internal/restic/snapshot_policy_test.go +++ b/internal/restic/snapshot_policy_test.go @@ -22,13 +22,14 @@ func parseTimeUTC(s string) time.Time { return t.UTC() } -func parseDuration(s string) restic.Duration { - d, err := restic.ParseDuration(s) - if err != nil { - panic(err) +// Returns the maximum number of snapshots to be kept according to this policy. +// If any of the counts is -1 it will return 0. +func policySum(e *restic.ExpirePolicy) int { + if e.Last == -1 || e.Hourly == -1 || e.Daily == -1 || e.Weekly == -1 || e.Monthly == -1 || e.Yearly == -1 { + return 0 } - return d + return e.Last + e.Hourly + e.Daily + e.Weekly + e.Monthly + e.Yearly } func TestExpireSnapshotOps(t *testing.T) { @@ -46,7 +47,7 @@ func TestExpireSnapshotOps(t *testing.T) { if isEmpty != d.expectEmpty { t.Errorf("empty test %v: wrong result, want:\n %#v\ngot:\n %#v", i, d.expectEmpty, isEmpty) } - hasSum := d.p.Sum() + hasSum := policySum(d.p) if hasSum != d.expectSum { t.Errorf("sum test %v: wrong result, want:\n %#v\ngot:\n %#v", i, d.expectSum, hasSum) } @@ -219,26 +220,30 @@ func TestApplyPolicy(t *testing.T) { {Tags: []restic.TagList{{"foo"}}}, {Tags: []restic.TagList{{"foo", "bar"}}}, {Tags: []restic.TagList{{"foo"}, {"bar"}}}, - {Within: parseDuration("1d")}, - {Within: parseDuration("2d")}, - {Within: parseDuration("7d")}, - {Within: parseDuration("1m")}, - {Within: parseDuration("1m14d")}, - {Within: parseDuration("1y1d1m")}, - {Within: parseDuration("13d23h")}, - {Within: parseDuration("2m2h")}, - {Within: parseDuration("1y2m3d3h")}, - {WithinHourly: parseDuration("1y2m3d3h")}, - {WithinDaily: parseDuration("1y2m3d3h")}, - {WithinWeekly: parseDuration("1y2m3d3h")}, - {WithinMonthly: parseDuration("1y2m3d3h")}, - {WithinYearly: parseDuration("1y2m3d3h")}, - {Within: parseDuration("1h"), - WithinHourly: parseDuration("1d"), - WithinDaily: parseDuration("7d"), - WithinWeekly: parseDuration("1m"), - WithinMonthly: parseDuration("1y"), - WithinYearly: parseDuration("9999y")}, + {Within: restic.ParseDurationOrPanic("1d")}, + {Within: restic.ParseDurationOrPanic("2d")}, + {Within: restic.ParseDurationOrPanic("7d")}, + {Within: restic.ParseDurationOrPanic("1m")}, + {Within: restic.ParseDurationOrPanic("1m14d")}, + {Within: restic.ParseDurationOrPanic("1y1d1m")}, + {Within: restic.ParseDurationOrPanic("13d23h")}, + {Within: restic.ParseDurationOrPanic("2m2h")}, + {Within: restic.ParseDurationOrPanic("1y2m3d3h")}, + {WithinHourly: restic.ParseDurationOrPanic("1y2m3d3h")}, + {WithinDaily: restic.ParseDurationOrPanic("1y2m3d3h")}, + {WithinWeekly: restic.ParseDurationOrPanic("1y2m3d3h")}, + {WithinMonthly: restic.ParseDurationOrPanic("1y2m3d3h")}, + {WithinYearly: restic.ParseDurationOrPanic("1y2m3d3h")}, + {Within: restic.ParseDurationOrPanic("1h"), + WithinHourly: restic.ParseDurationOrPanic("1d"), + WithinDaily: restic.ParseDurationOrPanic("7d"), + WithinWeekly: restic.ParseDurationOrPanic("1m"), + WithinMonthly: restic.ParseDurationOrPanic("1y"), + WithinYearly: restic.ParseDurationOrPanic("9999y")}, + {Last: -1}, // keep all + {Last: -1, Hourly: -1}, // keep all (Last overrides Hourly) + {Hourly: -1}, // keep all hourlies + {Daily: 3, Weekly: 2, Monthly: -1, Yearly: -1}, } for i, p := range tests { @@ -251,9 +256,9 @@ func TestApplyPolicy(t *testing.T) { len(keep)+len(remove), len(testExpireSnapshots)) } - if p.Sum() > 0 && len(keep) > p.Sum() { + if policySum(&p) > 0 && len(keep) > policySum(&p) { t.Errorf("not enough snapshots removed: policy allows %v snapshots to remain, but ended up with %v", - p.Sum(), len(keep)) + policySum(&p), len(keep)) } if len(keep) != len(reasons) { diff --git a/internal/restic/testdata/policy_keep_snapshots_36 b/internal/restic/testdata/policy_keep_snapshots_36 new file mode 100644 index 000000000..75a3a5b46 --- /dev/null +++ b/internal/restic/testdata/policy_keep_snapshots_36 @@ -0,0 +1,1782 @@ +{ + "keep": [ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": [ + "path1", + "path2" + ], + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2015-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-15T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2014-11-13T10:20:30.1Z", + "tree": null, + "paths": null, + "tags": [ + "bar" + ] + }, + { + "time": "2014-11-13T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-11-12T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-11-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-11-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-20T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-11T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-09T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-06T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-05T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-02T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-01T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null + } + ], + "reasons": [ + { + "snapshot": { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-01T01:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-11-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": [ + "path1", + "path2" + ], + "tags": [ + "foo", + "bar" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-11-15T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-11-13T10:20:30.1Z", + "tree": null, + "paths": null, + "tags": [ + "bar" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-11-13T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-11-12T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-11-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-11-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-10-20T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-10-11T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-10-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-10-09T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-10-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-10-06T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-10-05T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-10-02T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-10-01T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + } + ] +} \ No newline at end of file diff --git a/internal/restic/testdata/policy_keep_snapshots_37 b/internal/restic/testdata/policy_keep_snapshots_37 new file mode 100644 index 000000000..f6ffa40ea --- /dev/null +++ b/internal/restic/testdata/policy_keep_snapshots_37 @@ -0,0 +1,1872 @@ +{ + "keep": [ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": [ + "path1", + "path2" + ], + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2015-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-15T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2014-11-13T10:20:30.1Z", + "tree": null, + "paths": null, + "tags": [ + "bar" + ] + }, + { + "time": "2014-11-13T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-11-12T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-11-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-11-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-20T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-11T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-09T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-06T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-05T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-02T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-01T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null + } + ], + "reasons": [ + { + "snapshot": { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-01T01:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": [ + "path1", + "path2" + ], + "tags": [ + "foo", + "bar" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-15T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-13T10:20:30.1Z", + "tree": null, + "paths": null, + "tags": [ + "bar" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-13T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-12T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-20T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-11T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-09T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-06T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-05T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-02T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-01T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + } + ] +} \ No newline at end of file diff --git a/internal/restic/testdata/policy_keep_snapshots_38 b/internal/restic/testdata/policy_keep_snapshots_38 new file mode 100644 index 000000000..6bfdd57f1 --- /dev/null +++ b/internal/restic/testdata/policy_keep_snapshots_38 @@ -0,0 +1,1538 @@ +{ + "keep": [ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-15T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2014-11-13T10:20:30.1Z", + "tree": null, + "paths": null, + "tags": [ + "bar" + ] + }, + { + "time": "2014-11-12T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-11-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-11-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-20T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-11T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-09T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-06T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-05T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-02T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-01T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null + } + ], + "reasons": [ + { + "snapshot": { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-15T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-13T10:20:30.1Z", + "tree": null, + "paths": null, + "tags": [ + "bar" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-12T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-20T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-11T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-09T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-06T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-05T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-02T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-01T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + } + ] +} diff --git a/internal/restic/testdata/policy_keep_snapshots_39 b/internal/restic/testdata/policy_keep_snapshots_39 new file mode 100644 index 000000000..a8e6ca827 --- /dev/null +++ b/internal/restic/testdata/policy_keep_snapshots_39 @@ -0,0 +1,194 @@ +{ + "keep": [ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-22T10:20:30Z", + "tree": null, + "paths": null + } + ], + "reasons": [ + { + "snapshot": { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "daily snapshot", + "weekly snapshot", + "monthly snapshot", + "yearly snapshot" + ], + "counters": {"Daily": 2, "Weekly": 1, "Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "daily snapshot", + "weekly snapshot" + ], + "counters": {"Daily": 1, "Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "daily snapshot" + ], + "counters": {"Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "monthly snapshot", + "yearly snapshot" + ], + "counters": {"Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "monthly snapshot" + ], + "counters": {"Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "monthly snapshot" + ], + "counters": {"Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "monthly snapshot" + ], + "counters": {"Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "monthly snapshot", + "yearly snapshot" + ], + "counters": {"Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2014-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "monthly snapshot" + ], + "counters": {"Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2014-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "monthly snapshot" + ], + "counters": {"Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2014-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "monthly snapshot" + ], + "counters": {"Monthly": -1, "Yearly": -1} + } + ] +} \ No newline at end of file diff --git a/internal/restic/testing.go b/internal/restic/testing.go index ebafdf651..dda9eff44 100644 --- a/internal/restic/testing.go +++ b/internal/restic/testing.go @@ -212,3 +212,14 @@ func TestParseHandle(s string, t BlobType) BlobHandle { func TestSetSnapshotID(t testing.TB, sn *Snapshot, id ID) { sn.id = &id } + +// ParseDurationOrPanic parses a duration from a string or panics if string is invalid. +// The format is `6y5m234d37h`. +func ParseDurationOrPanic(s string) Duration { + d, err := ParseDuration(s) + if err != nil { + panic(err) + } + + return d +} diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index 2deef1cd2..3bb7489ba 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -12,6 +12,7 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/restore" ) // TODO if a blob is corrupt, there may be good blob copies in other packs @@ -54,6 +55,7 @@ type fileRestorer struct { filesWriter *filesWriter zeroChunk restic.ID sparse bool + progress *restore.Progress dst string files []*fileInfo @@ -65,7 +67,8 @@ func newFileRestorer(dst string, key *crypto.Key, idx func(restic.BlobHandle) []restic.PackedBlob, connections uint, - sparse bool) *fileRestorer { + sparse bool, + progress *restore.Progress) *fileRestorer { // as packs are streamed the concurrency is limited by IO workerCount := int(connections) @@ -77,6 +80,7 @@ func newFileRestorer(dst string, filesWriter: newFilesWriter(workerCount), zeroChunk: repository.ZeroChunk(), sparse: sparse, + progress: progress, workerCount: workerCount, dst: dst, Error: restorerAbortOnAllErrors, @@ -177,6 +181,8 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { wg.Go(func() error { for _, id := range packOrder { pack := packs[id] + // allow garbage collection of packInfo + delete(packs, id) select { case <-ctx.Done(): return ctx.Err() @@ -268,7 +274,13 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { file.inProgress = true createSize = file.size } - return r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize, file.sparse) + writeErr := r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize, file.sparse) + + if r.progress != nil { + r.progress.AddProgress(file.location, uint64(len(blobData)), uint64(file.size)) + } + + return writeErr } err := sanitizeError(file, writeToFile()) if err != nil { diff --git a/internal/restorer/filerestorer_test.go b/internal/restorer/filerestorer_test.go index b39afa249..e798f2b8b 100644 --- a/internal/restorer/filerestorer_test.go +++ b/internal/restorer/filerestorer_test.go @@ -150,7 +150,7 @@ func newTestRepo(content []TestFile) *TestRepo { func restoreAndVerify(t *testing.T, tempdir string, content []TestFile, files map[string]bool, sparse bool) { repo := newTestRepo(content) - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, sparse) + r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, sparse, nil) if files == nil { r.files = repo.files @@ -265,7 +265,7 @@ func TestErrorRestoreFiles(t *testing.T) { return loadError } - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false) + r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false, nil) r.files = repo.files err := r.restoreFiles(context.TODO()) @@ -304,7 +304,7 @@ func testPartialDownloadError(t *testing.T, part int) { return loader(ctx, h, length, offset, fn) } - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false) + r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false, nil) r.files = repo.files r.Error = func(s string, e error) error { // ignore errors as in the `restore` command diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 4dfe3c3a8..4acd45f95 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -10,6 +10,7 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" + restoreui "github.com/restic/restic/internal/ui/restore" "golang.org/x/sync/errgroup" ) @@ -20,6 +21,8 @@ type Restorer struct { sn *restic.Snapshot sparse bool + progress *restoreui.Progress + Error func(location string, err error) error SelectFilter func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) } @@ -27,12 +30,14 @@ type Restorer struct { var restorerAbortOnAllErrors = func(location string, err error) error { return err } // NewRestorer creates a restorer preloaded with the content from the snapshot id. -func NewRestorer(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, sparse bool) *Restorer { +func NewRestorer(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, sparse bool, + progress *restoreui.Progress) *Restorer { r := &Restorer{ repo: repo, sparse: sparse, Error: restorerAbortOnAllErrors, SelectFilter: func(string, string, *restic.Node) (bool, bool) { return true, true }, + progress: progress, sn: sn, } @@ -161,12 +166,14 @@ func (res *Restorer) restoreNodeTo(ctx context.Context, node *restic.Node, targe err := node.CreateAt(ctx, target, res.repo) if err != nil { debug.Log("node.CreateAt(%s) error %v", target, err) - } - if err == nil { - err = res.restoreNodeMetadataTo(node, target, location) + return err } - return err + if res.progress != nil { + res.progress.AddProgress(location, 0, 0) + } + + return res.restoreNodeMetadataTo(node, target, location) } func (res *Restorer) restoreNodeMetadataTo(node *restic.Node, target, location string) error { @@ -186,6 +193,11 @@ func (res *Restorer) restoreHardlinkAt(node *restic.Node, target, path, location if err != nil { return errors.WithStack(err) } + + if res.progress != nil { + res.progress.AddProgress(location, 0, 0) + } + // TODO investigate if hardlinks have separate metadata on any supported system return res.restoreNodeMetadataTo(node, path, location) } @@ -200,6 +212,10 @@ func (res *Restorer) restoreEmptyFileAt(node *restic.Node, target, location stri return err } + if res.progress != nil { + res.progress.AddProgress(location, 0, 0) + } + return res.restoreNodeMetadataTo(node, target, location) } @@ -215,7 +231,8 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { } idx := NewHardlinkIndex() - filerestorer := newFileRestorer(dst, res.repo.Backend().Load, res.repo.Key(), res.repo.Index().Lookup, res.repo.Connections(), res.sparse) + filerestorer := newFileRestorer(dst, res.repo.Backend().Load, res.repo.Key(), res.repo.Index().Lookup, + res.repo.Connections(), res.sparse, res.progress) filerestorer.Error = res.Error debug.Log("first pass for %q", dst) @@ -224,6 +241,9 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { _, err = res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{ enterDir: func(node *restic.Node, target, location string) error { debug.Log("first pass, enterDir: mkdir %q, leaveDir should restore metadata", location) + if res.progress != nil { + res.progress.AddFile(0) + } // create dir with default permissions // #leaveDir restores dir metadata after visiting all children return fs.MkdirAll(target, 0700) @@ -239,20 +259,34 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { } if node.Type != "file" { + if res.progress != nil { + res.progress.AddFile(0) + } return nil } if node.Size == 0 { + if res.progress != nil { + res.progress.AddFile(node.Size) + } return nil // deal with empty files later } if node.Links > 1 { if idx.Has(node.Inode, node.DeviceID) { + if res.progress != nil { + // a hardlinked file does not increase the restore size + res.progress.AddFile(0) + } return nil } idx.Add(node.Inode, node.DeviceID, location) } + if res.progress != nil { + res.progress.AddFile(node.Size) + } + filerestorer.addFile(location, node.Content, int64(node.Size)) return nil @@ -291,7 +325,13 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { return res.restoreNodeMetadataTo(node, target, location) }, - leaveDir: res.restoreNodeMetadataTo, + leaveDir: func(node *restic.Node, target, location string) error { + err := res.restoreNodeMetadataTo(node, target, location) + if err == nil && res.progress != nil { + res.progress.AddProgress(location, 0, 0) + } + return err + }, }) return err } diff --git a/internal/restorer/restorer_test.go b/internal/restorer/restorer_test.go index d6cd0c80a..1b0883bbb 100644 --- a/internal/restorer/restorer_test.go +++ b/internal/restorer/restorer_test.go @@ -325,7 +325,7 @@ func TestRestorer(t *testing.T) { sn, id := saveSnapshot(t, repo, test.Snapshot) t.Logf("snapshot saved as %v", id.Str()) - res := NewRestorer(context.TODO(), repo, sn, false) + res := NewRestorer(context.TODO(), repo, sn, false, nil) tempdir := rtest.TempDir(t) // make sure we're creating a new subdir of the tempdir @@ -442,7 +442,7 @@ func TestRestorerRelative(t *testing.T) { sn, id := saveSnapshot(t, repo, test.Snapshot) t.Logf("snapshot saved as %v", id.Str()) - res := NewRestorer(context.TODO(), repo, sn, false) + res := NewRestorer(context.TODO(), repo, sn, false, nil) tempdir := rtest.TempDir(t) cleanup := rtest.Chdir(t, tempdir) @@ -671,7 +671,7 @@ func TestRestorerTraverseTree(t *testing.T) { repo := repository.TestRepository(t) sn, _ := saveSnapshot(t, repo, test.Snapshot) - res := NewRestorer(context.TODO(), repo, sn, false) + res := NewRestorer(context.TODO(), repo, sn, false, nil) res.SelectFilter = test.Select @@ -747,7 +747,7 @@ func TestRestorerConsistentTimestampsAndPermissions(t *testing.T) { }, }) - res := NewRestorer(context.TODO(), repo, sn, false) + res := NewRestorer(context.TODO(), repo, sn, false, nil) res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { switch filepath.ToSlash(item) { @@ -802,7 +802,7 @@ func TestVerifyCancel(t *testing.T) { repo := repository.TestRepository(t) sn, _ := saveSnapshot(t, repo, snapshot) - res := NewRestorer(context.TODO(), repo, sn, false) + res := NewRestorer(context.TODO(), repo, sn, false, nil) tempdir := rtest.TempDir(t) ctx, cancel := context.WithCancel(context.Background()) @@ -844,7 +844,7 @@ func TestRestorerSparseFiles(t *testing.T) { archiver.SnapshotOptions{}) rtest.OK(t, err) - res := NewRestorer(context.TODO(), repo, sn, true) + res := NewRestorer(context.TODO(), repo, sn, true, nil) tempdir := rtest.TempDir(t) ctx, cancel := context.WithCancel(context.Background()) diff --git a/internal/restorer/restorer_unix_test.go b/internal/restorer/restorer_unix_test.go index dc327a9c9..4c5f2a5b8 100644 --- a/internal/restorer/restorer_unix_test.go +++ b/internal/restorer/restorer_unix_test.go @@ -9,10 +9,12 @@ import ( "path/filepath" "syscall" "testing" + "time" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" + restoreui "github.com/restic/restic/internal/ui/restore" ) func TestRestorerRestoreEmptyHardlinkedFileds(t *testing.T) { @@ -29,7 +31,7 @@ func TestRestorerRestoreEmptyHardlinkedFileds(t *testing.T) { }, }) - res := NewRestorer(context.TODO(), repo, sn, false) + res := NewRestorer(context.TODO(), repo, sn, false, nil) res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { return true, true @@ -66,3 +68,56 @@ func getBlockCount(t *testing.T, filename string) int64 { } return st.Blocks } + +type printerMock struct { + filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64 +} + +func (p *printerMock) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { +} +func (p *printerMock) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { + p.filesFinished = filesFinished + p.filesTotal = filesTotal + p.allBytesWritten = allBytesWritten + p.allBytesTotal = allBytesTotal +} + +func TestRestorerProgressBar(t *testing.T) { + repo := repository.TestRepository(t) + + sn, _ := saveSnapshot(t, repo, Snapshot{ + Nodes: map[string]Node{ + "dirtest": Dir{ + Nodes: map[string]Node{ + "file1": File{Links: 2, Inode: 1, Data: "foo"}, + "file2": File{Links: 2, Inode: 1, Data: "foo"}, + }, + }, + "file2": File{Links: 1, Inode: 2, Data: "example"}, + }, + }) + + mock := &printerMock{} + progress := restoreui.NewProgress(mock, 0) + res := NewRestorer(context.TODO(), repo, sn, false, progress) + res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { + return true, true + } + + tempdir := rtest.TempDir(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) + progress.Finish() + + const filesFinished = 4 + const filesTotal = filesFinished + const allBytesWritten = 10 + const allBytesTotal = allBytesWritten + rtest.Assert(t, mock.filesFinished == filesFinished, "filesFinished: expected %v, got %v", filesFinished, mock.filesFinished) + rtest.Assert(t, mock.filesTotal == filesTotal, "filesTotal: expected %v, got %v", filesTotal, mock.filesTotal) + rtest.Assert(t, mock.allBytesWritten == allBytesWritten, "allBytesWritten: expected %v, got %v", allBytesWritten, mock.allBytesWritten) + rtest.Assert(t, mock.allBytesTotal == allBytesTotal, "allBytesTotal: expected %v, got %v", allBytesTotal, mock.allBytesTotal) +} diff --git a/internal/ui/backup/text.go b/internal/ui/backup/text.go index 0c5f897dd..acb2a8d3a 100644 --- a/internal/ui/backup/text.go +++ b/internal/ui/backup/text.go @@ -86,6 +86,8 @@ func (b *TextProgress) Error(item string, err error) error { // CompleteItem is the status callback function for the archiver when a // file/dir has been saved successfully. func (b *TextProgress) CompleteItem(messageType, item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration) { + item = termstatus.Quote(item) + switch messageType { case "dir new": b.VV("new %v, saved in %.3fs (%v added, %v stored, %v metadata)", diff --git a/internal/ui/restore/progressformatter.go b/internal/ui/restore/progressformatter.go new file mode 100644 index 000000000..a89cc628e --- /dev/null +++ b/internal/ui/restore/progressformatter.go @@ -0,0 +1,131 @@ +package restore + +import ( + "fmt" + "sync" + "time" + + "github.com/restic/restic/internal/ui" + "github.com/restic/restic/internal/ui/progress" +) + +type Progress struct { + updater progress.Updater + m sync.Mutex + + progressInfoMap map[string]progressInfoEntry + filesFinished uint64 + filesTotal uint64 + allBytesWritten uint64 + allBytesTotal uint64 + started time.Time + + printer ProgressPrinter +} + +type progressInfoEntry struct { + bytesWritten uint64 + bytesTotal uint64 +} + +type ProgressPrinter interface { + Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) + Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) +} + +func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress { + p := &Progress{ + progressInfoMap: make(map[string]progressInfoEntry), + started: time.Now(), + printer: printer, + } + p.updater = *progress.NewUpdater(interval, p.update) + return p +} + +func (p *Progress) update(runtime time.Duration, final bool) { + p.m.Lock() + defer p.m.Unlock() + + if !final { + p.printer.Update(p.filesFinished, p.filesTotal, p.allBytesWritten, p.allBytesTotal, runtime) + } else { + p.printer.Finish(p.filesFinished, p.filesTotal, p.allBytesWritten, p.allBytesTotal, runtime) + } +} + +// AddFile starts tracking a new file with the given size +func (p *Progress) AddFile(size uint64) { + p.m.Lock() + defer p.m.Unlock() + + p.filesTotal++ + p.allBytesTotal += size +} + +// AddProgress accumulates the number of bytes written for a file +func (p *Progress) AddProgress(name string, bytesWrittenPortion uint64, bytesTotal uint64) { + p.m.Lock() + defer p.m.Unlock() + + entry, exists := p.progressInfoMap[name] + if !exists { + entry.bytesTotal = bytesTotal + } + entry.bytesWritten += bytesWrittenPortion + p.progressInfoMap[name] = entry + + p.allBytesWritten += bytesWrittenPortion + if entry.bytesWritten == entry.bytesTotal { + delete(p.progressInfoMap, name) + p.filesFinished++ + } +} + +func (p *Progress) Finish() { + p.updater.Done() +} + +type term interface { + Print(line string) + SetStatus(lines []string) +} + +type textPrinter struct { + terminal term +} + +func NewProgressPrinter(terminal term) ProgressPrinter { + return &textPrinter{ + terminal: terminal, + } +} + +func (t *textPrinter) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { + timeLeft := ui.FormatDuration(duration) + formattedAllBytesWritten := ui.FormatBytes(allBytesWritten) + formattedAllBytesTotal := ui.FormatBytes(allBytesTotal) + allPercent := ui.FormatPercent(allBytesWritten, allBytesTotal) + progress := fmt.Sprintf("[%s] %s %v files %s, total %v files %v", + timeLeft, allPercent, filesFinished, formattedAllBytesWritten, filesTotal, formattedAllBytesTotal) + + t.terminal.SetStatus([]string{progress}) +} + +func (t *textPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { + t.terminal.SetStatus([]string{}) + + timeLeft := ui.FormatDuration(duration) + formattedAllBytesTotal := ui.FormatBytes(allBytesTotal) + + var summary string + if filesFinished == filesTotal && allBytesWritten == allBytesTotal { + summary = fmt.Sprintf("Summary: Restored %d Files (%s) in %s", filesTotal, formattedAllBytesTotal, timeLeft) + } else { + formattedAllBytesWritten := ui.FormatBytes(allBytesWritten) + summary = fmt.Sprintf("Summary: Restored %d / %d Files (%s / %s) in %s", + filesFinished, filesTotal, formattedAllBytesWritten, formattedAllBytesTotal, timeLeft) + } + + t.terminal.Print(summary) +} diff --git a/internal/ui/restore/progressformatter_test.go b/internal/ui/restore/progressformatter_test.go new file mode 100644 index 000000000..0cc4ea1ba --- /dev/null +++ b/internal/ui/restore/progressformatter_test.go @@ -0,0 +1,170 @@ +package restore + +import ( + "testing" + "time" + + "github.com/restic/restic/internal/test" +) + +type printerTraceEntry struct { + filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64 + + duration time.Duration + isFinished bool +} + +type printerTrace []printerTraceEntry + +type mockPrinter struct { + trace printerTrace +} + +const mockFinishDuration = 42 * time.Second + +func (p *mockPrinter) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { + p.trace = append(p.trace, printerTraceEntry{filesFinished, filesTotal, allBytesWritten, allBytesTotal, duration, false}) +} +func (p *mockPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { + p.trace = append(p.trace, printerTraceEntry{filesFinished, filesTotal, allBytesWritten, allBytesTotal, mockFinishDuration, true}) +} + +func testProgress(fn func(progress *Progress) bool) printerTrace { + printer := &mockPrinter{} + progress := NewProgress(printer, 0) + final := fn(progress) + progress.update(0, final) + trace := append(printerTrace{}, printer.trace...) + // cleanup to avoid goroutine leak, but copy trace first + progress.Finish() + return trace +} + +func TestNew(t *testing.T) { + result := testProgress(func(progress *Progress) bool { + return false + }) + test.Equals(t, printerTrace{ + printerTraceEntry{0, 0, 0, 0, 0, false}, + }, result) +} + +func TestAddFile(t *testing.T) { + fileSize := uint64(100) + + result := testProgress(func(progress *Progress) bool { + progress.AddFile(fileSize) + return false + }) + test.Equals(t, printerTrace{ + printerTraceEntry{0, 1, 0, fileSize, 0, false}, + }, result) +} + +func TestFirstProgressOnAFile(t *testing.T) { + expectedBytesWritten := uint64(5) + expectedBytesTotal := uint64(100) + + result := testProgress(func(progress *Progress) bool { + progress.AddFile(expectedBytesTotal) + progress.AddProgress("test", expectedBytesWritten, expectedBytesTotal) + return false + }) + test.Equals(t, printerTrace{ + printerTraceEntry{0, 1, expectedBytesWritten, expectedBytesTotal, 0, false}, + }, result) +} + +func TestLastProgressOnAFile(t *testing.T) { + fileSize := uint64(100) + + result := testProgress(func(progress *Progress) bool { + progress.AddFile(fileSize) + progress.AddProgress("test", 30, fileSize) + progress.AddProgress("test", 35, fileSize) + progress.AddProgress("test", 35, fileSize) + return false + }) + test.Equals(t, printerTrace{ + printerTraceEntry{1, 1, fileSize, fileSize, 0, false}, + }, result) +} + +func TestLastProgressOnLastFile(t *testing.T) { + fileSize := uint64(100) + + result := testProgress(func(progress *Progress) bool { + progress.AddFile(fileSize) + progress.AddFile(50) + progress.AddProgress("test1", 50, 50) + progress.AddProgress("test2", 50, fileSize) + progress.AddProgress("test2", 50, fileSize) + return false + }) + test.Equals(t, printerTrace{ + printerTraceEntry{2, 2, 50 + fileSize, 50 + fileSize, 0, false}, + }, result) +} + +func TestSummaryOnSuccess(t *testing.T) { + fileSize := uint64(100) + + result := testProgress(func(progress *Progress) bool { + progress.AddFile(fileSize) + progress.AddFile(50) + progress.AddProgress("test1", 50, 50) + progress.AddProgress("test2", fileSize, fileSize) + return true + }) + test.Equals(t, printerTrace{ + printerTraceEntry{2, 2, 50 + fileSize, 50 + fileSize, mockFinishDuration, true}, + }, result) +} + +func TestSummaryOnErrors(t *testing.T) { + fileSize := uint64(100) + + result := testProgress(func(progress *Progress) bool { + progress.AddFile(fileSize) + progress.AddFile(50) + progress.AddProgress("test1", 50, 50) + progress.AddProgress("test2", fileSize/2, fileSize) + return true + }) + test.Equals(t, printerTrace{ + printerTraceEntry{1, 2, 50 + fileSize/2, 50 + fileSize, mockFinishDuration, true}, + }, result) +} + +type mockTerm struct { + output []string +} + +func (m *mockTerm) Print(line string) { + m.output = append(m.output, line) +} + +func (m *mockTerm) SetStatus(lines []string) { + m.output = append([]string{}, lines...) +} + +func TestPrintUpdate(t *testing.T) { + term := &mockTerm{} + printer := NewProgressPrinter(term) + printer.Update(3, 11, 29, 47, 5*time.Second) + test.Equals(t, []string{"[0:05] 61.70% 3 files 29 B, total 11 files 47 B"}, term.output) +} + +func TestPrintSummaryOnSuccess(t *testing.T) { + term := &mockTerm{} + printer := NewProgressPrinter(term) + printer.Finish(11, 11, 47, 47, 5*time.Second) + test.Equals(t, []string{"Summary: Restored 11 Files (47 B) in 0:05"}, term.output) +} + +func TestPrintSummaryOnErrors(t *testing.T) { + term := &mockTerm{} + printer := NewProgressPrinter(term) + printer.Finish(3, 11, 29, 47, 5*time.Second) + test.Equals(t, []string{"Summary: Restored 3 / 11 Files (29 B / 47 B) in 0:05"}, term.output) +} diff --git a/internal/ui/termstatus/status.go b/internal/ui/termstatus/status.go index fdc7e14f6..5b310ec80 100644 --- a/internal/ui/termstatus/status.go +++ b/internal/ui/termstatus/status.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "os" + "strconv" "strings" "unicode" @@ -302,29 +303,54 @@ func Truncate(s string, w int) string { return s } - for i, r := range s { + for i := uint(0); i < uint(len(s)); { + utfsize := uint(1) // UTF-8 encoding size of first rune in s. w-- - if r > unicode.MaxASCII && wideRune(r) { - w-- + + if s[i] > unicode.MaxASCII { + var wide bool + if wide, utfsize = wideRune(s[i:]); wide { + w-- + } } if w < 0 { return s[:i] } + i += utfsize } return s } -// Guess whether r would occupy two terminal cells instead of one. -// This cannot be determined exactly without knowing the terminal font, -// so we treat all ambigous runes as full-width, i.e., two cells. -func wideRune(r rune) bool { - kind := width.LookupRune(r).Kind() - return kind != width.Neutral && kind != width.EastAsianNarrow +// Guess whether the first rune in s would occupy two terminal cells +// instead of one. This cannot be determined exactly without knowing +// the terminal font, so we treat all ambigous runes as full-width, +// i.e., two cells. +func wideRune(s string) (wide bool, utfsize uint) { + prop, size := width.LookupString(s) + kind := prop.Kind() + wide = kind != width.Neutral && kind != width.EastAsianNarrow + return wide, uint(size) +} + +func sanitizeLines(lines []string, width int) []string { + // Sanitize lines and truncate them if they're too long. + for i, line := range lines { + line = Quote(line) + if width > 0 { + line = Truncate(line, width-2) + } + if i < len(lines)-1 { // Last line gets no line break. + line += "\n" + } + lines[i] = line + } + return lines } // SetStatus updates the status lines. +// The lines should not contain newlines; this method adds them. func (t *Terminal) SetStatus(lines []string) { if len(lines) == 0 { return @@ -341,21 +367,25 @@ func (t *Terminal) SetStatus(lines []string) { } } - // make sure that all lines have a line break and are not too long - for i, line := range lines { - line = strings.TrimRight(line, "\n") - if width > 0 { - line = Truncate(line, width-2) - } - lines[i] = line + "\n" - } - - // make sure the last line does not have a line break - last := len(lines) - 1 - lines[last] = strings.TrimRight(lines[last], "\n") + sanitizeLines(lines, width) select { case t.status <- status{lines: lines}: case <-t.closed: } } + +// Quote lines with funny characters in them, meaning control chars, newlines, +// tabs, anything else non-printable and invalid UTF-8. +// +// This is intended to produce a string that does not mess up the terminal +// rather than produce an unambiguous quoted string. +func Quote(line string) string { + for _, r := range line { + // The replacement character usually means the input is not UTF-8. + if r == unicode.ReplacementChar || !unicode.IsPrint(r) { + return strconv.Quote(line) + } + } + return line +} diff --git a/internal/ui/termstatus/status_test.go b/internal/ui/termstatus/status_test.go index ce18f42e6..b59063076 100644 --- a/internal/ui/termstatus/status_test.go +++ b/internal/ui/termstatus/status_test.go @@ -1,6 +1,78 @@ package termstatus -import "testing" +import ( + "bytes" + "context" + "fmt" + "io" + "strconv" + "testing" + + rtest "github.com/restic/restic/internal/test" +) + +func TestSetStatus(t *testing.T) { + var buf bytes.Buffer + term := New(&buf, io.Discard, false) + + term.canUpdateStatus = true + term.fd = ^uintptr(0) + term.clearCurrentLine = posixClearCurrentLine + term.moveCursorUp = posixMoveCursorUp + + ctx, cancel := context.WithCancel(context.Background()) + go term.Run(ctx) + + const ( + clear = posixControlClearLine + home = posixControlMoveCursorHome + up = posixControlMoveCursorUp + ) + + term.SetStatus([]string{"first"}) + exp := home + clear + "first" + home + + term.SetStatus([]string{"foo", "bar", "baz"}) + exp += home + clear + "foo\n" + home + clear + "bar\n" + + home + clear + "baz" + home + up + up + + term.SetStatus([]string{"quux", "needs\nquote"}) + exp += home + clear + "quux\n" + + home + clear + "\"needs\\nquote\"\n" + + home + clear + home + up + up // Third line implicit. + + cancel() + exp += home + clear + "\n" + home + clear + "\n" + + home + up + up // Status cleared. + + <-term.closed + rtest.Equals(t, exp, buf.String()) +} + +func TestQuote(t *testing.T) { + for _, c := range []struct { + in string + needQuote bool + }{ + {"foo.bar/baz", false}, + {"föó_bàŕ-bãẑ", false}, + {" foo ", false}, + {"foo bar", false}, + {"foo\nbar", true}, + {"foo\rbar", true}, + {"foo\abar", true}, + {"\xff", true}, + {`c:\foo\bar`, false}, + // Issue #2260: terminal control characters. + {"\x1bm_red_is_beautiful", true}, + } { + if c.needQuote { + rtest.Equals(t, strconv.Quote(c.in), Quote(c.in)) + } else { + rtest.Equals(t, c.in, Quote(c.in)) + } + } +} func TestTruncate(t *testing.T) { var tests = []struct { @@ -49,11 +121,35 @@ func BenchmarkTruncateASCII(b *testing.B) { func BenchmarkTruncateUnicode(b *testing.B) { s := "Hello World or Καλημέρα κόσμε or こんにちは 世界" w := 0 - for _, r := range s { + for i := 0; i < len(s); { w++ - if wideRune(r) { + wide, utfsize := wideRune(s[i:]) + if wide { w++ } + i += int(utfsize) } + b.ResetTimer() + benchmarkTruncate(b, s, w-1) } + +func TestSanitizeLines(t *testing.T) { + var tests = []struct { + input []string + width int + output []string + }{ + {[]string{""}, 80, []string{""}}, + {[]string{"too long test line"}, 10, []string{"too long"}}, + {[]string{"too long test line", "text"}, 10, []string{"too long\n", "text"}}, + {[]string{"too long test line", "second long test line"}, 10, []string{"too long\n", "second l"}}, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%s %d", test.input, test.width), func(t *testing.T) { + out := sanitizeLines(test.input, test.width) + rtest.Equals(t, test.output, out) + }) + } +} diff --git a/internal/walker/rewriter.go b/internal/walker/rewriter.go index 6f063831e..649857032 100644 --- a/internal/walker/rewriter.go +++ b/internal/walker/rewriter.go @@ -9,13 +9,47 @@ import ( "github.com/restic/restic/internal/restic" ) -// SelectByNameFunc returns true for all items that should be included (files and -// dirs). If false is returned, files are ignored and dirs are not even walked. -type SelectByNameFunc func(item string) bool +type NodeRewriteFunc func(node *restic.Node, path string) *restic.Node +type FailedTreeRewriteFunc func(nodeID restic.ID, path string, err error) (restic.ID, error) -type TreeFilterVisitor struct { - SelectByName SelectByNameFunc - PrintExclude func(string) +type RewriteOpts struct { + // return nil to remove the node + RewriteNode NodeRewriteFunc + // decide what to do with a tree that could not be loaded. Return nil to remove the node. By default the load error is returned which causes the operation to fail. + RewriteFailedTree FailedTreeRewriteFunc + + AllowUnstableSerialization bool + DisableNodeCache bool +} + +type idMap map[restic.ID]restic.ID + +type TreeRewriter struct { + opts RewriteOpts + + replaces idMap +} + +func NewTreeRewriter(opts RewriteOpts) *TreeRewriter { + rw := &TreeRewriter{ + opts: opts, + } + if !opts.DisableNodeCache { + rw.replaces = make(idMap) + } + // setup default implementations + if rw.opts.RewriteNode == nil { + rw.opts.RewriteNode = func(node *restic.Node, path string) *restic.Node { + return node + } + } + if rw.opts.RewriteFailedTree == nil { + // fail with error by default + rw.opts.RewriteFailedTree = func(nodeID restic.ID, path string, err error) (restic.ID, error) { + return restic.ID{}, err + } + } + return rw } type BlobLoadSaver interface { @@ -23,51 +57,58 @@ type BlobLoadSaver interface { restic.BlobLoader } -func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID restic.ID, visitor *TreeFilterVisitor) (newNodeID restic.ID, err error) { - curTree, err := restic.LoadTree(ctx, repo, nodeID) - if err != nil { - return restic.ID{}, err +func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID restic.ID) (newNodeID restic.ID, err error) { + // check if tree was already changed + newID, ok := t.replaces[nodeID] + if ok { + return newID, nil } - // check that we can properly encode this tree without losing information - // The alternative of using json/Decoder.DisallowUnknownFields() doesn't work as we use - // a custom UnmarshalJSON to decode trees, see also https://github.com/golang/go/issues/41144 - testID, err := restic.SaveTree(ctx, repo, curTree) + // a nil nodeID will lead to a load error + curTree, err := restic.LoadTree(ctx, repo, nodeID) if err != nil { - return restic.ID{}, err + return t.opts.RewriteFailedTree(nodeID, nodepath, err) } - if nodeID != testID { - return restic.ID{}, fmt.Errorf("cannot encode tree at %q without loosing information", nodepath) + + if !t.opts.AllowUnstableSerialization { + // check that we can properly encode this tree without losing information + // The alternative of using json/Decoder.DisallowUnknownFields() doesn't work as we use + // a custom UnmarshalJSON to decode trees, see also https://github.com/golang/go/issues/41144 + testID, err := restic.SaveTree(ctx, repo, curTree) + if err != nil { + return restic.ID{}, err + } + if nodeID != testID { + return restic.ID{}, fmt.Errorf("cannot encode tree at %q without losing information", nodepath) + } } debug.Log("filterTree: %s, nodeId: %s\n", nodepath, nodeID.Str()) - changed := false tb := restic.NewTreeJSONBuilder() for _, node := range curTree.Nodes { path := path.Join(nodepath, node.Name) - if !visitor.SelectByName(path) { - if visitor.PrintExclude != nil { - visitor.PrintExclude(path) - } - changed = true + node = t.opts.RewriteNode(node, path) + if node == nil { continue } - if node.Subtree == nil { + if node.Type != "dir" { err = tb.AddNode(node) if err != nil { return restic.ID{}, err } continue } - newID, err := FilterTree(ctx, repo, path, *node.Subtree, visitor) + // treat nil as null id + var subtree restic.ID + if node.Subtree != nil { + subtree = *node.Subtree + } + newID, err := t.RewriteTree(ctx, repo, path, subtree) if err != nil { return restic.ID{}, err } - if !node.Subtree.Equal(newID) { - changed = true - } node.Subtree = &newID err = tb.AddNode(node) if err != nil { @@ -75,17 +116,18 @@ func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID } } - if changed { - tree, err := tb.Finalize() - if err != nil { - return restic.ID{}, err - } - - // Save new tree - newTreeID, _, _, err := repo.SaveBlob(ctx, restic.TreeBlob, tree, restic.ID{}, false) - debug.Log("filterTree: save new tree for %s as %v\n", nodepath, newTreeID) - return newTreeID, err + tree, err := tb.Finalize() + if err != nil { + return restic.ID{}, err } - return nodeID, nil + // Save new tree + newTreeID, _, _, err := repo.SaveBlob(ctx, restic.TreeBlob, tree, restic.ID{}, false) + if t.replaces != nil { + t.replaces[nodeID] = newTreeID + } + if !newTreeID.Equal(nodeID) { + debug.Log("filterTree: save new tree for %s as %v\n", nodepath, newTreeID) + } + return newTreeID, err } diff --git a/internal/walker/rewriter_test.go b/internal/walker/rewriter_test.go index 3dcf0ac9e..07ce5f72f 100644 --- a/internal/walker/rewriter_test.go +++ b/internal/walker/rewriter_test.go @@ -5,9 +5,9 @@ import ( "fmt" "testing" - "github.com/google/go-cmp/cmp" "github.com/pkg/errors" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" ) // WritableTreeMap also support saving @@ -38,26 +38,26 @@ func (t WritableTreeMap) Dump() { } } -type checkRewriteFunc func(t testing.TB) (visitor TreeFilterVisitor, final func(testing.TB)) +type checkRewriteFunc func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) // checkRewriteItemOrder ensures that the order of the 'path' arguments is the one passed in as 'want'. func checkRewriteItemOrder(want []string) checkRewriteFunc { pos := 0 - return func(t testing.TB) (visitor TreeFilterVisitor, final func(testing.TB)) { - vis := TreeFilterVisitor{ - SelectByName: func(path string) bool { + return func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) { + rewriter = NewTreeRewriter(RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { if pos >= len(want) { t.Errorf("additional unexpected path found: %v", path) - return false + return nil } if path != want[pos] { t.Errorf("wrong path found, want %q, got %q", want[pos], path) } pos++ - return true + return node }, - } + }) final = func(t testing.TB) { if pos != len(want) { @@ -65,21 +65,20 @@ func checkRewriteItemOrder(want []string) checkRewriteFunc { } } - return vis, final + return rewriter, final } } -// checkRewriteSkips excludes nodes if path is in skipFor, it checks that all excluded entries are printed. -func checkRewriteSkips(skipFor map[string]struct{}, want []string) checkRewriteFunc { +// checkRewriteSkips excludes nodes if path is in skipFor, it checks that rewriting proceedes in the correct order. +func checkRewriteSkips(skipFor map[string]struct{}, want []string, disableCache bool) checkRewriteFunc { var pos int - printed := make(map[string]struct{}) - return func(t testing.TB) (visitor TreeFilterVisitor, final func(testing.TB)) { - vis := TreeFilterVisitor{ - SelectByName: func(path string) bool { + return func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) { + rewriter = NewTreeRewriter(RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { if pos >= len(want) { t.Errorf("additional unexpected path found: %v", path) - return false + return nil } if path != want[pos] { @@ -87,27 +86,40 @@ func checkRewriteSkips(skipFor map[string]struct{}, want []string) checkRewriteF } pos++ - _, ok := skipFor[path] - return !ok - }, - PrintExclude: func(s string) { - if _, ok := printed[s]; ok { - t.Errorf("path was already printed %v", s) + _, skip := skipFor[path] + if skip { + return nil } - printed[s] = struct{}{} + return node }, - } + DisableNodeCache: disableCache, + }) final = func(t testing.TB) { - if !cmp.Equal(skipFor, printed) { - t.Errorf("unexpected paths skipped: %s", cmp.Diff(skipFor, printed)) - } if pos != len(want) { t.Errorf("not enough items returned, want %d, got %d", len(want), pos) } } - return vis, final + return rewriter, final + } +} + +// checkIncreaseNodeSize modifies each node by changing its size. +func checkIncreaseNodeSize(increase uint64) checkRewriteFunc { + return func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) { + rewriter = NewTreeRewriter(RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { + if node.Type == "file" { + node.Size += increase + } + return node + }, + }) + + final = func(t testing.TB) {} + + return rewriter, final } } @@ -150,6 +162,7 @@ func TestRewriter(t *testing.T) { "/subdir", "/subdir/subfile", }, + false, ), }, { // exclude dir @@ -170,6 +183,91 @@ func TestRewriter(t *testing.T) { "/foo", "/subdir", }, + false, + ), + }, + { // modify node + tree: TestTree{ + "foo": TestFile{Size: 21}, + "subdir": TestTree{ + "subfile": TestFile{Size: 21}, + }, + }, + newTree: TestTree{ + "foo": TestFile{Size: 42}, + "subdir": TestTree{ + "subfile": TestFile{Size: 42}, + }, + }, + check: checkIncreaseNodeSize(21), + }, + { // test cache + tree: TestTree{ + // both subdirs are identical + "subdir1": TestTree{ + "subfile": TestFile{}, + "subfile2": TestFile{}, + }, + "subdir2": TestTree{ + "subfile": TestFile{}, + "subfile2": TestFile{}, + }, + }, + newTree: TestTree{ + "subdir1": TestTree{ + "subfile2": TestFile{}, + }, + "subdir2": TestTree{ + "subfile2": TestFile{}, + }, + }, + check: checkRewriteSkips( + map[string]struct{}{ + "/subdir1/subfile": {}, + }, + []string{ + "/subdir1", + "/subdir1/subfile", + "/subdir1/subfile2", + "/subdir2", + }, + false, + ), + }, + { // test disabled cache + tree: TestTree{ + // both subdirs are identical + "subdir1": TestTree{ + "subfile": TestFile{}, + "subfile2": TestFile{}, + }, + "subdir2": TestTree{ + "subfile": TestFile{}, + "subfile2": TestFile{}, + }, + }, + newTree: TestTree{ + "subdir1": TestTree{ + "subfile2": TestFile{}, + }, + "subdir2": TestTree{ + "subfile": TestFile{}, + "subfile2": TestFile{}, + }, + }, + check: checkRewriteSkips( + map[string]struct{}{ + "/subdir1/subfile": {}, + }, + []string{ + "/subdir1", + "/subdir1/subfile", + "/subdir1/subfile2", + "/subdir2", + "/subdir2/subfile", + "/subdir2/subfile2", + }, + true, ), }, } @@ -186,8 +284,8 @@ func TestRewriter(t *testing.T) { ctx, cancel := context.WithCancel(context.TODO()) defer cancel() - vis, last := test.check(t) - newRoot, err := FilterTree(ctx, modrepo, "/", root, &vis) + rewriter, last := test.check(t) + newRoot, err := rewriter.RewriteTree(ctx, modrepo, "/", root) if err != nil { t.Error(err) } @@ -213,10 +311,56 @@ func TestRewriterFailOnUnknownFields(t *testing.T) { ctx, cancel := context.WithCancel(context.TODO()) defer cancel() - // use nil visitor to crash if the tree loading works unexpectedly - _, err := FilterTree(ctx, tm, "/", id, nil) + + rewriter := NewTreeRewriter(RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { + // tree loading must not succeed + t.Fail() + return node + }, + }) + _, err := rewriter.RewriteTree(ctx, tm, "/", id) if err == nil { t.Error("missing error on unknown field") } + + // check that the serialization check can be disabled + rewriter = NewTreeRewriter(RewriteOpts{ + AllowUnstableSerialization: true, + }) + root, err := rewriter.RewriteTree(ctx, tm, "/", id) + test.OK(t, err) + _, expRoot := BuildTreeMap(TestTree{ + "subfile": TestFile{}, + }) + test.Assert(t, root == expRoot, "mismatched trees") +} + +func TestRewriterTreeLoadError(t *testing.T) { + tm := WritableTreeMap{TreeMap{}} + id := restic.NewRandomID() + + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + + // also check that load error by default cause the operation to fail + rewriter := NewTreeRewriter(RewriteOpts{}) + _, err := rewriter.RewriteTree(ctx, tm, "/", id) + if err == nil { + t.Fatal("missing error on unloadable tree") + } + + replacementID := restic.NewRandomID() + rewriter = NewTreeRewriter(RewriteOpts{ + RewriteFailedTree: func(nodeID restic.ID, path string, err error) (restic.ID, error) { + if nodeID != id || path != "/" { + t.Fail() + } + return replacementID, nil + }, + }) + newRoot, err := rewriter.RewriteTree(ctx, tm, "/", id) + test.OK(t, err) + test.Equals(t, replacementID, newRoot) } diff --git a/internal/walker/walker_test.go b/internal/walker/walker_test.go index 6c4fd3436..8de1a9dc4 100644 --- a/internal/walker/walker_test.go +++ b/internal/walker/walker_test.go @@ -14,7 +14,9 @@ import ( type TestTree map[string]interface{} // TestNode is used to test the walker. -type TestFile struct{} +type TestFile struct { + Size uint64 +} func BuildTreeMap(tree TestTree) (m TreeMap, root restic.ID) { m = TreeMap{} @@ -37,6 +39,7 @@ func buildTreeMap(tree TestTree, m TreeMap) restic.ID { err := tb.AddNode(&restic.Node{ Name: name, Type: "file", + Size: elem.Size, }) if err != nil { panic(err)