From c1101ede198c28ea396231e3b330768defd680eb Mon Sep 17 00:00:00 2001 From: Niklas Wagner Date: Tue, 21 Feb 2023 13:52:58 +0100 Subject: [PATCH 001/127] Add jq to container image --- changelog/unreleased/pull-4220 | 5 +++++ docker/Dockerfile | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/pull-4220 diff --git a/changelog/unreleased/pull-4220 b/changelog/unreleased/pull-4220 new file mode 100644 index 000000000..787b6ba2d --- /dev/null +++ b/changelog/unreleased/pull-4220 @@ -0,0 +1,5 @@ +Enhancement: Add jq to container image + +The Docker container image now contains jq which can be useful when restic outputs json data. + +https://github.com/restic/restic/pull/4220 diff --git a/docker/Dockerfile b/docker/Dockerfile index 9f47fa10f..72fc85093 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,7 +11,7 @@ RUN go run build.go FROM alpine:latest AS restic -RUN apk add --update --no-cache ca-certificates fuse openssh-client tzdata +RUN apk add --update --no-cache ca-certificates fuse openssh-client tzdata jq COPY --from=builder /go/src/github.com/restic/restic/restic /usr/bin From 8161605f1b76d5978cf89deeb2b82cd6121c1c88 Mon Sep 17 00:00:00 2001 From: Torben Giesselmann Date: Fri, 3 Mar 2023 19:07:57 -0800 Subject: [PATCH 002/127] snapshot_group: Fix typo --- internal/restic/snapshot_group.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/restic/snapshot_group.go b/internal/restic/snapshot_group.go index c3f3307f6..9efae2ff6 100644 --- a/internal/restic/snapshot_group.go +++ b/internal/restic/snapshot_group.go @@ -68,7 +68,7 @@ type SnapshotGroupKey struct { } // GroupSnapshots takes a list of snapshots and a grouping criteria and creates -// a group list of snapshots. +// a grouped list of snapshots. func GroupSnapshots(snapshots Snapshots, groupBy SnapshotGroupByOptions) (map[string]Snapshots, bool, error) { // group by hostname and dirs snapshotGroups := make(map[string]Snapshots) From 6aca7dac21e08e86c30da7af3b490bf704455906 Mon Sep 17 00:00:00 2001 From: Torben Giesselmann Date: Fri, 3 Mar 2023 19:10:11 -0800 Subject: [PATCH 003/127] forget: Verify forget opts --- cmd/restic/cmd_forget.go | 34 +++++++++++++++++++++++- cmd/restic/cmd_forget_test.go | 50 +++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 cmd/restic/cmd_forget_test.go diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index fbe4c1c8a..caa205a45 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -99,8 +99,40 @@ func init() { addPruneOptions(cmdForget) } +func verifyForgetOptions(opts *ForgetOptions) error { + var negValFound = false + + if opts.Last < -1 || opts.Hourly < -1 || opts.Daily < -1 || opts.Weekly < -1 || + opts.Monthly < -1 || opts.Yearly < -1 { + negValFound = true + } + + if !negValFound { + // durations := [6]restic.Duration{opts.Within, opts.WithinHourly, opts.WithinDaily, + // opts.WithinMonthly, opts.WithinWeekly, opts.WithinYearly} + for _, d := range [6]restic.Duration{opts.Within, opts.WithinHourly, opts.WithinDaily, + opts.WithinMonthly, opts.WithinWeekly, opts.WithinYearly} { + if d.Hours < -1 || d.Days < -1 || d.Months < -1 || d.Years < -1 { + negValFound = true + break + } + } + } + + if negValFound { + return errors.Fatal("negative values other than -1 are not allowed for --keep-* options") + } + + return nil +} + func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, args []string) error { - err := verifyPruneOptions(&pruneOptions) + err := verifyForgetOptions(&opts) + if err != nil { + return err + } + + err = verifyPruneOptions(&pruneOptions) if err != nil { return err } diff --git a/cmd/restic/cmd_forget_test.go b/cmd/restic/cmd_forget_test.go new file mode 100644 index 000000000..778df4549 --- /dev/null +++ b/cmd/restic/cmd_forget_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "fmt" + "testing" + + "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" +) + +func TestPreventNegativeForgetOptionValues(t *testing.T) { + invalidForgetOpts := []ForgetOptions{ + {Last: -2}, + {Hourly: -2}, + {Daily: -2}, + {Weekly: -2}, + {Monthly: -2}, + {Yearly: -2}, + {Within: restic.Duration{Hours: -2}}, + {Within: restic.Duration{Days: -2}}, + {Within: restic.Duration{Months: -2}}, + {Within: restic.Duration{Years: -2}}, + {WithinHourly: restic.Duration{Hours: -2}}, + {WithinHourly: restic.Duration{Days: -2}}, + {WithinHourly: restic.Duration{Months: -2}}, + {WithinHourly: restic.Duration{Years: -2}}, + {WithinDaily: restic.Duration{Hours: -2}}, + {WithinDaily: restic.Duration{Days: -2}}, + {WithinDaily: restic.Duration{Months: -2}}, + {WithinDaily: restic.Duration{Years: -2}}, + {WithinWeekly: restic.Duration{Hours: -2}}, + {WithinWeekly: restic.Duration{Days: -2}}, + {WithinWeekly: restic.Duration{Months: -2}}, + {WithinWeekly: restic.Duration{Years: -2}}, + {WithinMonthly: restic.Duration{Hours: -2}}, + {WithinMonthly: restic.Duration{Days: -2}}, + {WithinMonthly: restic.Duration{Months: -2}}, + {WithinMonthly: restic.Duration{Years: -2}}, + {WithinYearly: restic.Duration{Hours: -2}}, + {WithinYearly: restic.Duration{Days: -2}}, + {WithinYearly: restic.Duration{Months: -2}}, + {WithinYearly: restic.Duration{Years: -2}}, + } + + for _, opts := range invalidForgetOpts { + err := verifyForgetOptions(&opts) + rtest.Assert(t, err != nil, fmt.Sprintf("should have returned error for %+v", opts)) + rtest.Equals(t, "Fatal: negative values other than -1 are not allowed for --keep-* options", err.Error()) + } +} From b77b0749fa44f8d60a4b7344802e2843a40dd72a Mon Sep 17 00:00:00 2001 From: Torben Giesselmann Date: Sat, 4 Mar 2023 19:01:37 -0800 Subject: [PATCH 004/127] forget: Treat -1 as forever for all "last n" opts --- cmd/restic/cmd_forget.go | 2 - internal/restic/snapshot_policy.go | 5 +- internal/restic/snapshot_policy_test.go | 6 + .../restic/testdata/policy_keep_snapshots_36 | 1782 ++++++++++++++++ .../restic/testdata/policy_keep_snapshots_37 | 1872 +++++++++++++++++ 5 files changed, 3664 insertions(+), 3 deletions(-) create mode 100644 internal/restic/testdata/policy_keep_snapshots_36 create mode 100644 internal/restic/testdata/policy_keep_snapshots_37 diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index caa205a45..2d9341fac 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -108,8 +108,6 @@ func verifyForgetOptions(opts *ForgetOptions) error { } if !negValFound { - // durations := [6]restic.Duration{opts.Within, opts.WithinHourly, opts.WithinDaily, - // opts.WithinMonthly, opts.WithinWeekly, opts.WithinYearly} for _, d := range [6]restic.Duration{opts.Within, opts.WithinHourly, opts.WithinDaily, opts.WithinMonthly, opts.WithinWeekly, opts.WithinYearly} { if d.Hours < -1 || d.Days < -1 || d.Months < -1 || d.Years < -1 { diff --git a/internal/restic/snapshot_policy.go b/internal/restic/snapshot_policy.go index 3271140aa..d63cf45e3 100644 --- a/internal/restic/snapshot_policy.go +++ b/internal/restic/snapshot_policy.go @@ -260,13 +260,16 @@ func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots, reason // Now update the other buckets and see if they have some counts left. for i, b := range buckets { - if b.Count > 0 { + if b.Count <= -1 || b.Count > 0 { val := b.bucker(cur.Time, nr) if val != b.Last { debug.Log("keep %v %v, bucker %v, val %v\n", cur.Time, cur.id.Str(), i, val) keepSnap = true buckets[i].Last = val buckets[i].Count-- + if buckets[i].Count < -1 { + buckets[i].Count = -1 + } keepSnapReasons = append(keepSnapReasons, b.reason) } } diff --git a/internal/restic/snapshot_policy_test.go b/internal/restic/snapshot_policy_test.go index 918ea4ec7..6f921f08e 100644 --- a/internal/restic/snapshot_policy_test.go +++ b/internal/restic/snapshot_policy_test.go @@ -239,6 +239,12 @@ func TestApplyPolicy(t *testing.T) { WithinWeekly: parseDuration("1m"), WithinMonthly: parseDuration("1y"), WithinYearly: parseDuration("9999y")}, + {Last: -1}, // keep all + {Last: -1, Hourly: -1}, // keep all (Last overrides Hourly) + // {Hourly: -1}, // keep all hourlies + // {Daily: -1}, // keep all dailies + // {Daily: -1, Weekly: 4}, // keep all dailies and 4 weeklies + // {Daily: 3, Weekly: -1}, // keep 3 dailies and all weeklies } for i, p := range tests { diff --git a/internal/restic/testdata/policy_keep_snapshots_36 b/internal/restic/testdata/policy_keep_snapshots_36 new file mode 100644 index 000000000..75a3a5b46 --- /dev/null +++ b/internal/restic/testdata/policy_keep_snapshots_36 @@ -0,0 +1,1782 @@ +{ + "keep": [ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": [ + "path1", + "path2" + ], + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2015-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-15T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2014-11-13T10:20:30.1Z", + "tree": null, + "paths": null, + "tags": [ + "bar" + ] + }, + { + "time": "2014-11-13T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-11-12T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-11-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-11-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-20T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-11T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-09T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-06T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-05T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-02T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-01T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null + } + ], + "reasons": [ + { + "snapshot": { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2016-01-01T01:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-11-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": [ + "path1", + "path2" + ], + "tags": [ + "foo", + "bar" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2015-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-11-15T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-11-13T10:20:30.1Z", + "tree": null, + "paths": null, + "tags": [ + "bar" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-11-13T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-11-12T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-11-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-11-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-10-20T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-10-11T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-10-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-10-09T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-10-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-10-06T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-10-05T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-10-02T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-10-01T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + }, + { + "snapshot": { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1} + } + ] +} \ No newline at end of file diff --git a/internal/restic/testdata/policy_keep_snapshots_37 b/internal/restic/testdata/policy_keep_snapshots_37 new file mode 100644 index 000000000..f6ffa40ea --- /dev/null +++ b/internal/restic/testdata/policy_keep_snapshots_37 @@ -0,0 +1,1872 @@ +{ + "keep": [ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": [ + "path1", + "path2" + ], + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2015-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-15T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2014-11-13T10:20:30.1Z", + "tree": null, + "paths": null, + "tags": [ + "bar" + ] + }, + { + "time": "2014-11-13T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-11-12T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-11-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-11-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-20T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-11T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-09T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-06T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-05T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-02T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-01T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null + } + ], + "reasons": [ + { + "snapshot": { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-01T01:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": [ + "path1", + "path2" + ], + "tags": [ + "foo", + "bar" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-15T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-13T10:20:30.1Z", + "tree": null, + "paths": null, + "tags": [ + "bar" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-13T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-12T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-20T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-11T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-09T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-06T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-05T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-02T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-01T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "last snapshot", + "hourly snapshot" + ], + "counters": {"Last": -1, "Hourly": -1} + } + ] +} \ No newline at end of file From 32e6a438be153aeb3d99f758646faedff8783c3e Mon Sep 17 00:00:00 2001 From: Torben Giesselmann Date: Sat, 4 Mar 2023 22:13:34 -0800 Subject: [PATCH 005/127] forget: Add test for "keep all hourly snapshots" --- internal/restic/snapshot_policy_test.go | 6 +- .../restic/testdata/policy_keep_snapshots_38 | 1538 +++++++++++++++++ 2 files changed, 1540 insertions(+), 4 deletions(-) create mode 100644 internal/restic/testdata/policy_keep_snapshots_38 diff --git a/internal/restic/snapshot_policy_test.go b/internal/restic/snapshot_policy_test.go index 6f921f08e..92e53848c 100644 --- a/internal/restic/snapshot_policy_test.go +++ b/internal/restic/snapshot_policy_test.go @@ -241,10 +241,8 @@ func TestApplyPolicy(t *testing.T) { WithinYearly: parseDuration("9999y")}, {Last: -1}, // keep all {Last: -1, Hourly: -1}, // keep all (Last overrides Hourly) - // {Hourly: -1}, // keep all hourlies - // {Daily: -1}, // keep all dailies - // {Daily: -1, Weekly: 4}, // keep all dailies and 4 weeklies - // {Daily: 3, Weekly: -1}, // keep 3 dailies and all weeklies + {Hourly: -1}, // keep all hourlies + // {Daily: 3, Weekly: 2, Monthly: -1, Yearly: -1}, } for i, p := range tests { diff --git a/internal/restic/testdata/policy_keep_snapshots_38 b/internal/restic/testdata/policy_keep_snapshots_38 new file mode 100644 index 000000000..6bfdd57f1 --- /dev/null +++ b/internal/restic/testdata/policy_keep_snapshots_38 @@ -0,0 +1,1538 @@ +{ + "keep": [ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-15T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2014-11-13T10:20:30.1Z", + "tree": null, + "paths": null, + "tags": [ + "bar" + ] + }, + { + "time": "2014-11-12T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-11-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-11-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-20T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-11T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-09T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-06T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-05T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-02T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-10-01T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null + } + ], + "reasons": [ + { + "snapshot": { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2015-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-15T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-13T10:20:30.1Z", + "tree": null, + "paths": null, + "tags": [ + "bar" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-12T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-11-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-20T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-11T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-10T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-09T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-08T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-06T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-05T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-02T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-10-01T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + }, + { + "snapshot": { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "hourly snapshot" + ], + "counters": {"Hourly": -1} + } + ] +} From ba183c44c3812882586083550d86ce782343be93 Mon Sep 17 00:00:00 2001 From: Torben Giesselmann Date: Sun, 5 Mar 2023 13:56:16 -0800 Subject: [PATCH 006/127] forget: Add test with regular and "forever" opts --- internal/restic/snapshot_policy_test.go | 10 +- .../restic/testdata/policy_keep_snapshots_39 | 194 ++++++++++++++++++ 2 files changed, 200 insertions(+), 4 deletions(-) create mode 100644 internal/restic/testdata/policy_keep_snapshots_39 diff --git a/internal/restic/snapshot_policy_test.go b/internal/restic/snapshot_policy_test.go index 92e53848c..97a01ce03 100644 --- a/internal/restic/snapshot_policy_test.go +++ b/internal/restic/snapshot_policy_test.go @@ -242,7 +242,7 @@ func TestApplyPolicy(t *testing.T) { {Last: -1}, // keep all {Last: -1, Hourly: -1}, // keep all (Last overrides Hourly) {Hourly: -1}, // keep all hourlies - // {Daily: 3, Weekly: 2, Monthly: -1, Yearly: -1}, + {Daily: 3, Weekly: 2, Monthly: -1, Yearly: -1}, } for i, p := range tests { @@ -255,9 +255,11 @@ func TestApplyPolicy(t *testing.T) { len(keep)+len(remove), len(testExpireSnapshots)) } - if p.Sum() > 0 && len(keep) > p.Sum() { - t.Errorf("not enough snapshots removed: policy allows %v snapshots to remain, but ended up with %v", - p.Sum(), len(keep)) + if p.Last >= 0 && p.Hourly >= 0 && p.Daily >= 0 && p.Weekly >= 0 && p.Monthly >= 0 && p.Yearly >= 0 { + if p.Sum() > 0 && len(keep) > p.Sum() { + t.Errorf("not enough snapshots removed: policy allows %v snapshots to remain, but ended up with %v", + p.Sum(), len(keep)) + } } if len(keep) != len(reasons) { diff --git a/internal/restic/testdata/policy_keep_snapshots_39 b/internal/restic/testdata/policy_keep_snapshots_39 new file mode 100644 index 000000000..a8e6ca827 --- /dev/null +++ b/internal/restic/testdata/policy_keep_snapshots_39 @@ -0,0 +1,194 @@ +{ + "keep": [ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + { + "time": "2014-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-22T10:20:30Z", + "tree": null, + "paths": null + } + ], + "reasons": [ + { + "snapshot": { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "daily snapshot", + "weekly snapshot", + "monthly snapshot", + "yearly snapshot" + ], + "counters": {"Daily": 2, "Weekly": 1, "Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "daily snapshot", + "weekly snapshot" + ], + "counters": {"Daily": 1, "Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + "matches": [ + "daily snapshot" + ], + "counters": {"Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "monthly snapshot", + "yearly snapshot" + ], + "counters": {"Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "monthly snapshot" + ], + "counters": {"Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "monthly snapshot" + ], + "counters": {"Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "monthly snapshot" + ], + "counters": {"Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "monthly snapshot", + "yearly snapshot" + ], + "counters": {"Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2014-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo" + ] + }, + "matches": [ + "monthly snapshot" + ], + "counters": {"Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2014-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "monthly snapshot" + ], + "counters": {"Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2014-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "monthly snapshot" + ], + "counters": {"Monthly": -1, "Yearly": -1} + } + ] +} \ No newline at end of file From 667536cea4074a81d787ba4ff6c711f6a22a060b Mon Sep 17 00:00:00 2001 From: Torben Giesselmann Date: Sun, 5 Mar 2023 14:18:08 -0800 Subject: [PATCH 007/127] forget: Allow neg. values in "--keep-within*" opts --- cmd/restic/cmd_forget.go | 10 ---------- cmd/restic/cmd_forget_test.go | 25 ------------------------- 2 files changed, 35 deletions(-) diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index 2d9341fac..82313c6ef 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -107,16 +107,6 @@ func verifyForgetOptions(opts *ForgetOptions) error { negValFound = true } - if !negValFound { - for _, d := range [6]restic.Duration{opts.Within, opts.WithinHourly, opts.WithinDaily, - opts.WithinMonthly, opts.WithinWeekly, opts.WithinYearly} { - if d.Hours < -1 || d.Days < -1 || d.Months < -1 || d.Years < -1 { - negValFound = true - break - } - } - } - if negValFound { return errors.Fatal("negative values other than -1 are not allowed for --keep-* options") } diff --git a/cmd/restic/cmd_forget_test.go b/cmd/restic/cmd_forget_test.go index 778df4549..c45d72815 100644 --- a/cmd/restic/cmd_forget_test.go +++ b/cmd/restic/cmd_forget_test.go @@ -4,7 +4,6 @@ import ( "fmt" "testing" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -16,30 +15,6 @@ func TestPreventNegativeForgetOptionValues(t *testing.T) { {Weekly: -2}, {Monthly: -2}, {Yearly: -2}, - {Within: restic.Duration{Hours: -2}}, - {Within: restic.Duration{Days: -2}}, - {Within: restic.Duration{Months: -2}}, - {Within: restic.Duration{Years: -2}}, - {WithinHourly: restic.Duration{Hours: -2}}, - {WithinHourly: restic.Duration{Days: -2}}, - {WithinHourly: restic.Duration{Months: -2}}, - {WithinHourly: restic.Duration{Years: -2}}, - {WithinDaily: restic.Duration{Hours: -2}}, - {WithinDaily: restic.Duration{Days: -2}}, - {WithinDaily: restic.Duration{Months: -2}}, - {WithinDaily: restic.Duration{Years: -2}}, - {WithinWeekly: restic.Duration{Hours: -2}}, - {WithinWeekly: restic.Duration{Days: -2}}, - {WithinWeekly: restic.Duration{Months: -2}}, - {WithinWeekly: restic.Duration{Years: -2}}, - {WithinMonthly: restic.Duration{Hours: -2}}, - {WithinMonthly: restic.Duration{Days: -2}}, - {WithinMonthly: restic.Duration{Months: -2}}, - {WithinMonthly: restic.Duration{Years: -2}}, - {WithinYearly: restic.Duration{Hours: -2}}, - {WithinYearly: restic.Duration{Days: -2}}, - {WithinYearly: restic.Duration{Months: -2}}, - {WithinYearly: restic.Duration{Years: -2}}, } for _, opts := range invalidForgetOpts { From cb5694d136aa77857f53b4c5d657eaa72b4f6b28 Mon Sep 17 00:00:00 2001 From: greatroar <61184462+greatroar@users.noreply.github.com> Date: Tue, 7 Mar 2023 22:12:08 +0100 Subject: [PATCH 008/127] fuse: Report fuse.Attr.Blocks correctly Fixes #4239. --- changelog/unreleased/issue-4239 | 11 +++++++++++ internal/fuse/file.go | 2 +- internal/fuse/fuse_test.go | 32 ++++++++++++++++++++++++++++++++ internal/fuse/link.go | 2 +- internal/fuse/snapshots_dir.go | 2 +- 5 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 changelog/unreleased/issue-4239 diff --git a/changelog/unreleased/issue-4239 b/changelog/unreleased/issue-4239 new file mode 100644 index 000000000..247f3d9ed --- /dev/null +++ b/changelog/unreleased/issue-4239 @@ -0,0 +1,11 @@ +Bugfix: Correct number of blocks reported in mount point + +Restic mount points incorrectly reported the number of 512-byte (POSIX +standard) blocks for files and links, due to a rounding bug. In particular, +empty files were reported as taking one block instead of zero. + +The rounding is now fixed: the number of blocks reported is the file size +(or link target size), divided by 512 and rounded up to a whole number. + +https://github.com/restic/restic/issues/4239 +https://github.com/restic/restic/pull/4240 diff --git a/internal/fuse/file.go b/internal/fuse/file.go index 28ff5d450..35bc2a73e 100644 --- a/internal/fuse/file.go +++ b/internal/fuse/file.go @@ -50,7 +50,7 @@ func (f *file) Attr(ctx context.Context, a *fuse.Attr) error { a.Inode = f.inode a.Mode = f.node.Mode a.Size = f.node.Size - a.Blocks = (f.node.Size / blockSize) + 1 + a.Blocks = (f.node.Size + blockSize - 1) / blockSize a.BlockSize = blockSize a.Nlink = uint32(f.node.Links) diff --git a/internal/fuse/fuse_test.go b/internal/fuse/fuse_test.go index e71bf6fee..863c7672d 100644 --- a/internal/fuse/fuse_test.go +++ b/internal/fuse/fuse_test.go @@ -8,6 +8,7 @@ import ( "context" "math/rand" "os" + "strings" "testing" "time" @@ -216,6 +217,37 @@ func testTopUIDGID(t *testing.T, cfg Config, repo restic.Repository, uid, gid ui rtest.Equals(t, uint32(0), attr.Gid) } +// Test reporting of fuse.Attr.Blocks in multiples of 512. +func TestBlocks(t *testing.T) { + root := &Root{} + + for _, c := range []struct { + size, blocks uint64 + }{ + {0, 0}, + {1, 1}, + {511, 1}, + {512, 1}, + {513, 2}, + {1024, 2}, + {1025, 3}, + {41253, 81}, + } { + target := strings.Repeat("x", int(c.size)) + + for _, n := range []fs.Node{ + &file{root: root, node: &restic.Node{Size: uint64(c.size)}}, + &link{root: root, node: &restic.Node{LinkTarget: target}}, + &snapshotLink{root: root, snapshot: &restic.Snapshot{}, target: target}, + } { + var a fuse.Attr + err := n.Attr(context.TODO(), &a) + rtest.OK(t, err) + rtest.Equals(t, c.blocks, a.Blocks) + } + } +} + func TestInodeFromNode(t *testing.T) { node := &restic.Node{Name: "foo.txt", Type: "chardev", Links: 2} ino1 := inodeFromNode(1, node) diff --git a/internal/fuse/link.go b/internal/fuse/link.go index f910aadc4..47ee666a3 100644 --- a/internal/fuse/link.go +++ b/internal/fuse/link.go @@ -42,7 +42,7 @@ func (l *link) Attr(ctx context.Context, a *fuse.Attr) error { a.Nlink = uint32(l.node.Links) a.Size = uint64(len(l.node.LinkTarget)) - a.Blocks = 1 + a.Size/blockSize + a.Blocks = (a.Size + blockSize - 1) / blockSize return nil } diff --git a/internal/fuse/snapshots_dir.go b/internal/fuse/snapshots_dir.go index 977d0ab17..c19155741 100644 --- a/internal/fuse/snapshots_dir.go +++ b/internal/fuse/snapshots_dir.go @@ -142,7 +142,7 @@ func (l *snapshotLink) Attr(ctx context.Context, a *fuse.Attr) error { a.Inode = l.inode a.Mode = os.ModeSymlink | 0777 a.Size = uint64(len(l.target)) - a.Blocks = 1 + a.Size/blockSize + a.Blocks = (a.Size + blockSize - 1) / blockSize a.Uid = l.root.uid a.Gid = l.root.gid a.Atime = l.snapshot.Time From a7ac9a4769c878bb055b9b3a365dc43cff10f14d Mon Sep 17 00:00:00 2001 From: greatroar <61184462+greatroar@users.noreply.github.com> Date: Mon, 13 Mar 2023 00:11:22 +0100 Subject: [PATCH 009/127] doc: Fix some links, focus less on implementation details The godoc for filepath.Match has the syntax, which is what is important for writing patterns. Use pkg.go.dev instead of golang.org/pkg. For #4245. Not all links fixed yet. --- doc/020_installation.rst | 4 ++-- doc/030_preparing_a_new_repo.rst | 6 +++--- doc/040_backup.rst | 10 +++++----- doc/070_encryption.rst | 2 +- doc/090_participating.rst | 4 ++-- doc/developer_information.rst | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/020_installation.rst b/doc/020_installation.rst index 5ae93c94d..0c6795a26 100644 --- a/doc/020_installation.rst +++ b/doc/020_installation.rst @@ -40,7 +40,7 @@ package from the official community repos, e.g. using ``apk``: Arch Linux ========== -On `Arch Linux `__, there is a package called ``restic`` +On `Arch Linux `__, there is a package called ``restic`` installed from the official community repos, e.g. with ``pacman -S``: .. code-block:: console @@ -271,7 +271,7 @@ From Source restic is written in the Go programming language and you need at least Go version 1.18. Building restic may also work with older versions of Go, but that's not supported. See the `Getting -started `__ guide of the Go project for +started `__ guide of the Go project for instructions how to install Go. In order to build restic from source, execute the following steps: diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index 39a3a0744..4ee110bce 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -290,7 +290,7 @@ like this: ``s3:http://server:port/bucket_name``. Minio Server ************ -`Minio `__ is an Open Source Object Storage, +`Minio `__ is an Open Source Object Storage, written in Go and compatible with Amazon S3 API. - Download and Install `Minio @@ -350,7 +350,7 @@ this command. Alibaba Cloud (Aliyun) Object Storage System (OSS) ************************************************** -`Alibaba OSS `__ is an +`Alibaba OSS `__ is an encrypted, secure, cost-effective, and easy-to-use object storage service that enables you to store, back up, and archive large amounts of data in the cloud. @@ -616,7 +616,7 @@ established. .. _service account: https://cloud.google.com/iam/docs/service-accounts .. _create a service account key: https://cloud.google.com/iam/docs/creating-managing-service-account-keys#iam-service-account-keys-create-console -.. _default authentication material: https://cloud.google.com/docs/authentication/production +.. _default authentication material: https://cloud.google.com/docs/authentication#service-accounts .. _other-services: diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 3b1a56bd6..d2079cdf6 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -253,14 +253,14 @@ This instructs restic to exclude files matching the following criteria: * All files matching ``*.go`` (second line in ``excludes.txt``) * All files and sub-directories named ``bar`` which reside somewhere below a directory called ``foo`` (fourth line in ``excludes.txt``) -Patterns use `filepath.Glob `__ internally, -see `filepath.Match `__ for -syntax. Patterns are tested against the full path of a file/dir to be saved, +Patterns use the syntax of the Go function +`filepath.Match `__ +and are tested against the full path of a file/dir to be saved, even if restic is passed a relative path to save. Empty lines and lines starting with a ``#`` are ignored. Environment variables in exclude files are expanded with `os.ExpandEnv -`__, so ``/home/$USER/foo`` will be +`__, so ``/home/$USER/foo`` will be expanded to ``/home/bob/foo`` for the user ``bob``. To get a literal dollar sign, write ``$$`` to the file - this has to be done even when there's no matching environment variable for the word following a single ``$``. Note @@ -380,7 +380,7 @@ contains one *pattern* per line. The file must be encoded as UTF-8, or UTF-16 with a byte-order mark. Leading and trailing whitespace is removed from the patterns. Empty lines and lines starting with a ``#`` are ignored and each pattern is expanded when read, such that special characters in it are expanded -using the Go function `filepath.Glob `__ +using the Go function `filepath.Glob `__ - please see its documentation for the syntax you can use in the patterns. The argument passed to ``--files-from-verbatim`` must be the name of a text file diff --git a/doc/070_encryption.rst b/doc/070_encryption.rst index a7b8716ac..dc651cc07 100644 --- a/doc/070_encryption.rst +++ b/doc/070_encryption.rst @@ -19,7 +19,7 @@ Encryption the implementation looks sane and I guess the deduplication trade-off is worth it. So… I’m going to use restic for my personal backups.*" `Filippo Valsorda`_ -.. _Filippo Valsorda: https://blog.filippo.io/restic-cryptography/ +.. _Filippo Valsorda: https://words.filippo.io/restic-cryptography/ ********************** Manage repository keys diff --git a/doc/090_participating.rst b/doc/090_participating.rst index 00a387974..890bd9018 100644 --- a/doc/090_participating.rst +++ b/doc/090_participating.rst @@ -33,8 +33,8 @@ The debug log will always contain all log messages restic generates. You can also instruct restic to print some or all debug messages to stderr. These can also be limited to e.g. a list of source files or a list of patterns for function names. The patterns are globbing patterns (see the -documentation for `path.Glob `__), multiple -patterns are separated by commas. Patterns are case sensitive. +documentation for `filepath.Match `__). +Multiple patterns are separated by commas. Patterns are case sensitive. Printing all log messages to the console can be achieved by setting the file filter to ``*``: diff --git a/doc/developer_information.rst b/doc/developer_information.rst index c05edc9d2..307851757 100644 --- a/doc/developer_information.rst +++ b/doc/developer_information.rst @@ -10,7 +10,7 @@ refer to the documentation for the respective version. The binary produced depends on the following things: * The source code for the release - * The exact version of the official `Go compiler `__ used to produce the binaries (running ``restic version`` will print this) + * The exact version of the official `Go compiler `__ used to produce the binaries (running ``restic version`` will print this) * The architecture and operating system the Go compiler runs on (Linux, ``amd64``) * The build tags (for official binaries, it's the tag ``selfupdate``) * The path where the source code is extracted to (``/restic``) From eaceaca1133eba06a542df7b5d358bed139ada06 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Mar 2023 02:07:39 +0000 Subject: [PATCH 010/127] build(deps): bump golang.org/x/oauth2 from 0.5.0 to 0.6.0 Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from 0.5.0 to 0.6.0. - [Release notes](https://github.com/golang/oauth2/releases) - [Commits](https://github.com/golang/oauth2/compare/v0.5.0...v0.6.0) --- updated-dependencies: - dependency-name: golang.org/x/oauth2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 08560c5bc..59acf89d0 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/spf13/pflag v1.0.5 golang.org/x/crypto v0.7.0 golang.org/x/net v0.8.0 - golang.org/x/oauth2 v0.5.0 + golang.org/x/oauth2 v0.6.0 golang.org/x/sync v0.1.0 golang.org/x/sys v0.6.0 golang.org/x/term v0.6.0 diff --git a/go.sum b/go.sum index d57448a81..9bc87fc85 100644 --- a/go.sum +++ b/go.sum @@ -192,8 +192,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= -golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From b7f03d01b8eed1c0dab111aa83ee9bfd1dd8fbeb Mon Sep 17 00:00:00 2001 From: Torben Giesselmann Date: Tue, 14 Mar 2023 19:16:24 -0700 Subject: [PATCH 011/127] Add helper function for Duration parsing Tests in cmd_forget_test.go need the same convenience function that was implemented in snapshot_policy_test.go. Function parseDuration(...) was moved to testing.go and renamed to ParseDurationOrPanic(...). --- internal/restic/testing.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/internal/restic/testing.go b/internal/restic/testing.go index ebafdf651..9c21b81d1 100644 --- a/internal/restic/testing.go +++ b/internal/restic/testing.go @@ -212,3 +212,14 @@ func TestParseHandle(s string, t BlobType) BlobHandle { func TestSetSnapshotID(t testing.TB, sn *Snapshot, id ID) { sn.id = &id } + +// Convenience function that parses a duration from a string or panics if string is invalid. +// The format is `6y5m234d37h`. +func ParseDurationOrPanic(s string) Duration { + d, err := ParseDuration(s) + if err != nil { + panic(err) + } + + return d +} From 84ede6ad7a4c8cc7791145b952e5f48d3c13266a Mon Sep 17 00:00:00 2001 From: Torben Giesselmann Date: Tue, 14 Mar 2023 19:20:03 -0700 Subject: [PATCH 012/127] forget: Prevent neg. values in --keep-within* opts --- cmd/restic/cmd_forget.go | 11 +++--- cmd/restic/cmd_forget_test.go | 66 ++++++++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 18 deletions(-) diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index 82313c6ef..ac3601584 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -100,15 +100,16 @@ func init() { } func verifyForgetOptions(opts *ForgetOptions) error { - var negValFound = false - if opts.Last < -1 || opts.Hourly < -1 || opts.Daily < -1 || opts.Weekly < -1 || opts.Monthly < -1 || opts.Yearly < -1 { - negValFound = true + return errors.Fatal("negative values other than -1 are not allowed for --keep-*") } - if negValFound { - return errors.Fatal("negative values other than -1 are not allowed for --keep-* options") + for _, d := range []restic.Duration{opts.Within, opts.WithinHourly, opts.WithinDaily, + opts.WithinMonthly, opts.WithinWeekly, opts.WithinYearly} { + if d.Hours < 0 || d.Days < 0 || d.Months < 0 || d.Years < 0 { + return errors.Fatal("durations containing negative values are not allowed for --keep-within*") + } } return nil diff --git a/cmd/restic/cmd_forget_test.go b/cmd/restic/cmd_forget_test.go index c45d72815..9fd5c7bb0 100644 --- a/cmd/restic/cmd_forget_test.go +++ b/cmd/restic/cmd_forget_test.go @@ -1,25 +1,65 @@ package main import ( - "fmt" "testing" + "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) -func TestPreventNegativeForgetOptionValues(t *testing.T) { - invalidForgetOpts := []ForgetOptions{ - {Last: -2}, - {Hourly: -2}, - {Daily: -2}, - {Weekly: -2}, - {Monthly: -2}, - {Yearly: -2}, +func TestForgetOptionValues(t *testing.T) { + const negValErrorMsg = "Fatal: negative values other than -1 are not allowed for --keep-*" + const negDurationValErrorMsg = "Fatal: durations containing negative values are not allowed for --keep-within*" + testCases := []struct { + input ForgetOptions + expectsError bool + errorMsg string + }{ + {ForgetOptions{Last: 1}, false, ""}, + {ForgetOptions{Hourly: 1}, false, ""}, + {ForgetOptions{Daily: 1}, false, ""}, + {ForgetOptions{Weekly: 1}, false, ""}, + {ForgetOptions{Monthly: 1}, false, ""}, + {ForgetOptions{Yearly: 1}, false, ""}, + {ForgetOptions{Last: 0}, false, ""}, + {ForgetOptions{Hourly: 0}, false, ""}, + {ForgetOptions{Daily: 0}, false, ""}, + {ForgetOptions{Weekly: 0}, false, ""}, + {ForgetOptions{Monthly: 0}, false, ""}, + {ForgetOptions{Yearly: 0}, false, ""}, + {ForgetOptions{Last: -1}, false, ""}, + {ForgetOptions{Hourly: -1}, false, ""}, + {ForgetOptions{Daily: -1}, false, ""}, + {ForgetOptions{Weekly: -1}, false, ""}, + {ForgetOptions{Monthly: -1}, false, ""}, + {ForgetOptions{Yearly: -1}, false, ""}, + {ForgetOptions{Last: -2}, true, negValErrorMsg}, + {ForgetOptions{Hourly: -2}, true, negValErrorMsg}, + {ForgetOptions{Daily: -2}, true, negValErrorMsg}, + {ForgetOptions{Weekly: -2}, true, negValErrorMsg}, + {ForgetOptions{Monthly: -2}, true, negValErrorMsg}, + {ForgetOptions{Yearly: -2}, true, negValErrorMsg}, + {ForgetOptions{Within: restic.ParseDurationOrPanic("1y2m3d3h")}, false, ""}, + {ForgetOptions{WithinHourly: restic.ParseDurationOrPanic("1y2m3d3h")}, false, ""}, + {ForgetOptions{WithinDaily: restic.ParseDurationOrPanic("1y2m3d3h")}, false, ""}, + {ForgetOptions{WithinWeekly: restic.ParseDurationOrPanic("1y2m3d3h")}, false, ""}, + {ForgetOptions{WithinMonthly: restic.ParseDurationOrPanic("2y4m6d8h")}, false, ""}, + {ForgetOptions{WithinYearly: restic.ParseDurationOrPanic("2y4m6d8h")}, false, ""}, + {ForgetOptions{Within: restic.ParseDurationOrPanic("-1y2m3d3h")}, true, negDurationValErrorMsg}, + {ForgetOptions{WithinHourly: restic.ParseDurationOrPanic("1y-2m3d3h")}, true, negDurationValErrorMsg}, + {ForgetOptions{WithinDaily: restic.ParseDurationOrPanic("1y2m-3d3h")}, true, negDurationValErrorMsg}, + {ForgetOptions{WithinWeekly: restic.ParseDurationOrPanic("1y2m3d-3h")}, true, negDurationValErrorMsg}, + {ForgetOptions{WithinMonthly: restic.ParseDurationOrPanic("-2y4m6d8h")}, true, negDurationValErrorMsg}, + {ForgetOptions{WithinYearly: restic.ParseDurationOrPanic("2y-4m6d8h")}, true, negDurationValErrorMsg}, } - for _, opts := range invalidForgetOpts { - err := verifyForgetOptions(&opts) - rtest.Assert(t, err != nil, fmt.Sprintf("should have returned error for %+v", opts)) - rtest.Equals(t, "Fatal: negative values other than -1 are not allowed for --keep-* options", err.Error()) + for _, testCase := range testCases { + err := verifyForgetOptions(&testCase.input) + if testCase.expectsError { + rtest.Assert(t, err != nil, "should have returned error for input %+v", testCase.input) + rtest.Equals(t, testCase.errorMsg, err.Error()) + } else { + rtest.Assert(t, err == nil, "expected no error for input %+v", testCase.input) + } } } From 1a584cb16e4df2782996bbfa5b9f4493485ffb46 Mon Sep 17 00:00:00 2001 From: Torben Giesselmann Date: Tue, 14 Mar 2023 19:29:08 -0700 Subject: [PATCH 013/127] Refactor policy sum calculation & duration parsing - Convert policy sum calculation to function and move to tests. - Remove parseDuration(...) and use ParseDurationOrPanic(...) instead. --- internal/restic/snapshot_policy.go | 16 ++----- internal/restic/snapshot_policy_test.go | 61 ++++++++++++------------- 2 files changed, 35 insertions(+), 42 deletions(-) diff --git a/internal/restic/snapshot_policy.go b/internal/restic/snapshot_policy.go index d63cf45e3..a9ec4a9f7 100644 --- a/internal/restic/snapshot_policy.go +++ b/internal/restic/snapshot_policy.go @@ -100,13 +100,7 @@ func (e ExpirePolicy) String() (s string) { return s } -// Sum returns the maximum number of snapshots to be kept according to this -// policy. -func (e ExpirePolicy) Sum() int { - return e.Last + e.Hourly + e.Daily + e.Weekly + e.Monthly + e.Yearly -} - -// Empty returns true iff no policy has been configured (all values zero). +// Empty returns true if no policy has been configured (all values zero). func (e ExpirePolicy) Empty() bool { if len(e.Tags) != 0 { return false @@ -260,15 +254,15 @@ func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots, reason // Now update the other buckets and see if they have some counts left. for i, b := range buckets { - if b.Count <= -1 || b.Count > 0 { + // -1 means "keep all" + if b.Count > 0 || b.Count == -1 { val := b.bucker(cur.Time, nr) if val != b.Last { debug.Log("keep %v %v, bucker %v, val %v\n", cur.Time, cur.id.Str(), i, val) keepSnap = true buckets[i].Last = val - buckets[i].Count-- - if buckets[i].Count < -1 { - buckets[i].Count = -1 + if buckets[i].Count > 0 { + buckets[i].Count-- } keepSnapReasons = append(keepSnapReasons, b.reason) } diff --git a/internal/restic/snapshot_policy_test.go b/internal/restic/snapshot_policy_test.go index 97a01ce03..75f0f18f4 100644 --- a/internal/restic/snapshot_policy_test.go +++ b/internal/restic/snapshot_policy_test.go @@ -22,13 +22,14 @@ func parseTimeUTC(s string) time.Time { return t.UTC() } -func parseDuration(s string) restic.Duration { - d, err := restic.ParseDuration(s) - if err != nil { - panic(err) +// Returns the maximum number of snapshots to be kept according to this policy. +// If any of the counts is -1 it will return 0. +func policySum(e *restic.ExpirePolicy) int { + if e.Last == -1 || e.Hourly == -1 || e.Daily == -1 || e.Weekly == -1 || e.Monthly == -1 || e.Yearly == -1 { + return 0 } - return d + return e.Last + e.Hourly + e.Daily + e.Weekly + e.Monthly + e.Yearly } func TestExpireSnapshotOps(t *testing.T) { @@ -46,7 +47,7 @@ func TestExpireSnapshotOps(t *testing.T) { if isEmpty != d.expectEmpty { t.Errorf("empty test %v: wrong result, want:\n %#v\ngot:\n %#v", i, d.expectEmpty, isEmpty) } - hasSum := d.p.Sum() + hasSum := policySum(d.p) if hasSum != d.expectSum { t.Errorf("sum test %v: wrong result, want:\n %#v\ngot:\n %#v", i, d.expectSum, hasSum) } @@ -219,26 +220,26 @@ func TestApplyPolicy(t *testing.T) { {Tags: []restic.TagList{{"foo"}}}, {Tags: []restic.TagList{{"foo", "bar"}}}, {Tags: []restic.TagList{{"foo"}, {"bar"}}}, - {Within: parseDuration("1d")}, - {Within: parseDuration("2d")}, - {Within: parseDuration("7d")}, - {Within: parseDuration("1m")}, - {Within: parseDuration("1m14d")}, - {Within: parseDuration("1y1d1m")}, - {Within: parseDuration("13d23h")}, - {Within: parseDuration("2m2h")}, - {Within: parseDuration("1y2m3d3h")}, - {WithinHourly: parseDuration("1y2m3d3h")}, - {WithinDaily: parseDuration("1y2m3d3h")}, - {WithinWeekly: parseDuration("1y2m3d3h")}, - {WithinMonthly: parseDuration("1y2m3d3h")}, - {WithinYearly: parseDuration("1y2m3d3h")}, - {Within: parseDuration("1h"), - WithinHourly: parseDuration("1d"), - WithinDaily: parseDuration("7d"), - WithinWeekly: parseDuration("1m"), - WithinMonthly: parseDuration("1y"), - WithinYearly: parseDuration("9999y")}, + {Within: restic.ParseDurationOrPanic("1d")}, + {Within: restic.ParseDurationOrPanic("2d")}, + {Within: restic.ParseDurationOrPanic("7d")}, + {Within: restic.ParseDurationOrPanic("1m")}, + {Within: restic.ParseDurationOrPanic("1m14d")}, + {Within: restic.ParseDurationOrPanic("1y1d1m")}, + {Within: restic.ParseDurationOrPanic("13d23h")}, + {Within: restic.ParseDurationOrPanic("2m2h")}, + {Within: restic.ParseDurationOrPanic("1y2m3d3h")}, + {WithinHourly: restic.ParseDurationOrPanic("1y2m3d3h")}, + {WithinDaily: restic.ParseDurationOrPanic("1y2m3d3h")}, + {WithinWeekly: restic.ParseDurationOrPanic("1y2m3d3h")}, + {WithinMonthly: restic.ParseDurationOrPanic("1y2m3d3h")}, + {WithinYearly: restic.ParseDurationOrPanic("1y2m3d3h")}, + {Within: restic.ParseDurationOrPanic("1h"), + WithinHourly: restic.ParseDurationOrPanic("1d"), + WithinDaily: restic.ParseDurationOrPanic("7d"), + WithinWeekly: restic.ParseDurationOrPanic("1m"), + WithinMonthly: restic.ParseDurationOrPanic("1y"), + WithinYearly: restic.ParseDurationOrPanic("9999y")}, {Last: -1}, // keep all {Last: -1, Hourly: -1}, // keep all (Last overrides Hourly) {Hourly: -1}, // keep all hourlies @@ -255,11 +256,9 @@ func TestApplyPolicy(t *testing.T) { len(keep)+len(remove), len(testExpireSnapshots)) } - if p.Last >= 0 && p.Hourly >= 0 && p.Daily >= 0 && p.Weekly >= 0 && p.Monthly >= 0 && p.Yearly >= 0 { - if p.Sum() > 0 && len(keep) > p.Sum() { - t.Errorf("not enough snapshots removed: policy allows %v snapshots to remain, but ended up with %v", - p.Sum(), len(keep)) - } + if policySum(&p) > 0 && len(keep) > policySum(&p) { + t.Errorf("not enough snapshots removed: policy allows %v snapshots to remain, but ended up with %v", + policySum(&p), len(keep)) } if len(keep) != len(reasons) { From 5069c9edd95d5a13b71d682c8214962802df80e7 Mon Sep 17 00:00:00 2001 From: Torben Giesselmann Date: Wed, 15 Mar 2023 15:07:51 -0700 Subject: [PATCH 014/127] Represent -1 as "all" in ExpirePolicy's Stringer --- internal/restic/snapshot_policy.go | 33 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/internal/restic/snapshot_policy.go b/internal/restic/snapshot_policy.go index a9ec4a9f7..228e4c88a 100644 --- a/internal/restic/snapshot_policy.go +++ b/internal/restic/snapshot_policy.go @@ -31,23 +31,22 @@ func (e ExpirePolicy) String() (s string) { var keeps []string var keepw []string - if e.Last > 0 { - keeps = append(keeps, fmt.Sprintf("%d latest", e.Last)) - } - if e.Hourly > 0 { - keeps = append(keeps, fmt.Sprintf("%d hourly", e.Hourly)) - } - if e.Daily > 0 { - keeps = append(keeps, fmt.Sprintf("%d daily", e.Daily)) - } - if e.Weekly > 0 { - keeps = append(keeps, fmt.Sprintf("%d weekly", e.Weekly)) - } - if e.Monthly > 0 { - keeps = append(keeps, fmt.Sprintf("%d monthly", e.Monthly)) - } - if e.Yearly > 0 { - keeps = append(keeps, fmt.Sprintf("%d yearly", e.Yearly)) + for _, opt := range []struct { + count int + descr string + }{ + {e.Last, "latest"}, + {e.Hourly, "hourly"}, + {e.Daily, "daily"}, + {e.Weekly, "weekly"}, + {e.Monthly, "monthly"}, + {e.Yearly, "yearly"}, + } { + if opt.count > 0 { + keeps = append(keeps, fmt.Sprintf("%d %s", opt.count, opt.descr)) + } else if opt.count == -1 { + keeps = append(keeps, fmt.Sprintf("all %s", opt.descr)) + } } if !e.WithinHourly.Zero() { From 5dccab701adf082d777bf3816b033150c5be0932 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Mar 2023 02:18:58 +0000 Subject: [PATCH 015/127] build(deps): bump actions/setup-go from 3 to 4 Bumps [actions/setup-go](https://github.com/actions/setup-go) from 3 to 4. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/setup-go dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6b8819aca..b2e74a453 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -58,7 +58,7 @@ jobs: steps: - name: Set up Go ${{ matrix.go }} - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: ${{ matrix.go }} @@ -222,7 +222,7 @@ jobs: steps: - name: Set up Go ${{ env.latest_go }} - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: ${{ env.latest_go }} @@ -247,7 +247,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Go ${{ env.latest_go }} - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: ${{ env.latest_go }} From a0885d5d694b7619c03ee68558a0785a48651dfb Mon Sep 17 00:00:00 2001 From: greatroar <61184462+greatroar@users.noreply.github.com> Date: Tue, 21 Mar 2023 17:33:18 +0100 Subject: [PATCH 016/127] fuse: Mix inode hashes in a non-symmetric way Since 0.15 (#4020), inodes are generated as hashes of names, xor'd with the parent inode. That means that the inode of a/b/b is h(a/b/b) = h(a) ^ h(b) ^ h(b) = h(a). I.e., the grandchild has the same inode as the grandparent. GNU find trips over this because it thinks it has encountered a loop in the filesystem, and fails to search a/b/b. This happens more generally when the same name occurs an even number of times. Fix this by multiplying the parent by a large prime, so the combining operation is not longer symmetric in its arguments. This is what the FNV hash does, which we used prior to 0.15. The hash is now h(a/b/b) = h(b) ^ p*(h(b) ^ p*h(a)) Note that we already ensure that h(x) is never zero. Collisions can still occur, but they should be much less likely to occur within a single path. Fixes #4253. --- changelog/unreleased/issue-4253 | 18 ++++++++++++++++++ internal/fuse/fuse_test.go | 11 +++++++++++ internal/fuse/inode.go | 6 ++++-- 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/issue-4253 diff --git a/changelog/unreleased/issue-4253 b/changelog/unreleased/issue-4253 new file mode 100644 index 000000000..2471eab0b --- /dev/null +++ b/changelog/unreleased/issue-4253 @@ -0,0 +1,18 @@ +Bugfix: Mount command should no longer create spurious filesystem loops + +When a backup contains a directory that has the same name as its parent, +say, a/b/b, and the GNU find command were run on this backup in a restic +mount, find command would refuse to traverse the lowest "b" directory, +instead printing "File system loop detected". This is due to the way the +restic mount command generates inode numbers for directories in the mount +point. + +The rule for generating these inode numbers was changed in 0.15.0. It has +now been changed again to avoid this issue. A perfect rule does not exist, +but the probability of this behavior occurring is now extremely small. +When it does occur, the mount point is not broken, and scripts that traverse +the mount point should work as long as they don't rely on inode numbers for +detecting filesystem loops. + +https://github.com/restic/restic/issues/4253 +https://github.com/restic/restic/pull/4255 diff --git a/internal/fuse/fuse_test.go b/internal/fuse/fuse_test.go index e71bf6fee..ae0aaccab 100644 --- a/internal/fuse/fuse_test.go +++ b/internal/fuse/fuse_test.go @@ -226,6 +226,17 @@ func TestInodeFromNode(t *testing.T) { ino1 = inodeFromNode(1, node) ino2 = inodeFromNode(2, node) rtest.Assert(t, ino1 != ino2, "same inode %d but different parent", ino1) + + // Regression test: in a path a/b/b, the grandchild should not get the + // same inode as the grandparent. + a := &restic.Node{Name: "a", Type: "dir", Links: 2} + ab := &restic.Node{Name: "b", Type: "dir", Links: 2} + abb := &restic.Node{Name: "b", Type: "dir", Links: 2} + inoA := inodeFromNode(1, a) + inoAb := inodeFromNode(inoA, ab) + inoAbb := inodeFromNode(inoAb, abb) + rtest.Assert(t, inoA != inoAb, "inode(a/b) = inode(a)") + rtest.Assert(t, inoA != inoAbb, "inode(a/b/b) = inode(a)") } var sink uint64 diff --git a/internal/fuse/inode.go b/internal/fuse/inode.go index de975b167..5e2ece4ac 100644 --- a/internal/fuse/inode.go +++ b/internal/fuse/inode.go @@ -10,9 +10,11 @@ import ( "github.com/restic/restic/internal/restic" ) +const prime = 11400714785074694791 // prime1 from xxhash. + // inodeFromName generates an inode number for a file in a meta dir. func inodeFromName(parent uint64, name string) uint64 { - inode := parent ^ xxhash.Sum64String(cleanupNodeName(name)) + inode := prime*parent ^ xxhash.Sum64String(cleanupNodeName(name)) // Inode 0 is invalid and 1 is the root. Remap those. if inode < 2 { @@ -33,7 +35,7 @@ func inodeFromNode(parent uint64, node *restic.Node) (inode uint64) { } else { // Else, use the name and the parent inode. // node.{DeviceID,Inode} may not even be reliable. - inode = parent ^ xxhash.Sum64String(cleanupNodeName(node.Name)) + inode = prime*parent ^ xxhash.Sum64String(cleanupNodeName(node.Name)) } // Inode 0 is invalid and 1 is the root. Remap those. From 1f12915b0cf68d3f1187af7ffe85db0dbad5bc29 Mon Sep 17 00:00:00 2001 From: GuitarBilly <38086529+GuitarBilly@users.noreply.github.com> Date: Tue, 21 Mar 2023 21:00:05 +0100 Subject: [PATCH 017/127] Update 110_talks.rst Update 110_talks.rst align date to 2nd of April as agreed. ( match podcast publication date ) --- doc/110_talks.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/110_talks.rst b/doc/110_talks.rst index 06952896f..51dcd3e35 100644 --- a/doc/110_talks.rst +++ b/doc/110_talks.rst @@ -17,6 +17,8 @@ Talks The following talks will be or have been given about restic: +- 2021-04-02: `The Changelog: Restic has your backup + (Podcast) `__ - 2016-01-31: Lightning Talk at the Go Devroom at FOSDEM 2016, Brussels, Belgium - 2016-01-29: `restic - Backups mal From 0666fa11b8d365778d863c872ec66cc83a679df8 Mon Sep 17 00:00:00 2001 From: "Leo R. Lundgren" Date: Wed, 22 Mar 2023 17:37:58 +0100 Subject: [PATCH 018/127] doc: Correct broken links --- doc/020_installation.rst | 2 +- doc/030_preparing_a_new_repo.rst | 19 +++++++++---------- doc/110_talks.rst | 6 +++--- doc/manual_rest.rst | 2 +- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/doc/020_installation.rst b/doc/020_installation.rst index 0c6795a26..4488e31f9 100644 --- a/doc/020_installation.rst +++ b/doc/020_installation.rst @@ -93,7 +93,7 @@ You may also install it using `MacPorts `__: Nix & NixOS =========== -If you are using `Nix `__ or `NixOS `__ +If you are using `Nix / NixOS `__ there is a package available named ``restic``. It can be installed using ``nix-env``: diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index 0eede3e38..44223c146 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -273,7 +273,7 @@ For an S3-compatible server that is not Amazon (like Minio, see below), or is only available via HTTP, you can specify the URL to the server like this: ``s3:http://server:port/bucket_name``. -.. note:: restic expects `path-style URLs `__ +.. note:: restic expects `path-style URLs `__ like for example ``s3.us-west-2.amazonaws.com/bucket_name``. Virtual-hosted–style URLs like ``bucket_name.s3.us-west-2.amazonaws.com``, where the bucket name is part of the hostname are not supported. These must @@ -290,12 +290,11 @@ like this: ``s3:http://server:port/bucket_name``. Minio Server ************ -`Minio `__ is an Open Source Object Storage, +`Minio `__ is an Open Source Object Storage, written in Go and compatible with Amazon S3 API. -- Download and Install `Minio - Server `__. -- You can also refer to https://docs.minio.io for step by step guidance +- Download and Install `Minio Download `__. +- You can also refer to `Minio Docs `__ for step by step guidance on installation and getting started on Minio Client and Minio Server. You must first setup the following environment variables with the @@ -358,7 +357,7 @@ of data in the cloud. Alibaba OSS is S3 compatible so it can be used as a storage provider for a restic repository with a couple of extra parameters. -- Determine the correct `Alibaba OSS region endpoint `__ - this will be something like ``oss-eu-west-1.aliyuncs.com`` +- Determine the correct `Alibaba OSS region endpoint `__ - this will be something like ``oss-eu-west-1.aliyuncs.com`` - You'll need the region name too - this will be something like ``oss-eu-west-1`` You must first setup the following environment variables with the @@ -441,7 +440,7 @@ the naming convention of those variables follows the official Python Swift clien Restic should be compatible with an `OpenStack RC file -`__ +`__ in most cases. Once environment variables are set up, a new repository can be created. The @@ -614,8 +613,8 @@ The number of concurrent connections to the GCS service can be set with the ``-o gs.connections=10`` switch. By default, at most five parallel connections are established. -.. _service account: https://cloud.google.com/iam/docs/service-accounts -.. _create a service account key: https://cloud.google.com/iam/docs/creating-managing-service-account-keys#iam-service-account-keys-create-console +.. _service account: https://cloud.google.com/iam/docs/service-account-overview +.. _create a service account key: https://cloud.google.com/iam/docs/keys-create-delete .. _default authentication material: https://cloud.google.com/docs/authentication#service-accounts .. _other-services: @@ -748,7 +747,7 @@ Password prompt on Windows At the moment, restic only supports the default Windows console interaction. If you use emulation environments like -`MSYS2 `__ or +`MSYS2 `__ or `Cygwin `__, which use terminals like ``Mintty`` or ``rxvt``, you may get a password error. diff --git a/doc/110_talks.rst b/doc/110_talks.rst index 51dcd3e35..e32cda62a 100644 --- a/doc/110_talks.rst +++ b/doc/110_talks.rst @@ -26,11 +26,11 @@ The following talks will be or have been given about restic: Public lecture in German at `CCC Cologne e.V. `__ in Cologne, Germany - 2015-08-23: `A Solution to the Backup - Inconvenience `__: - Lecture at `FROSCON 2015 `__ in Bonn, Germany + Inconvenience `__: + Lecture at `FROSCON 2015 `__ in Bonn, Germany - 2015-02-01: `Lightning Talk at FOSDEM 2015 `__: A short introduction (with slightly outdated command line) - 2015-01-27: `Talk about restic at CCC - Aachen `__ + Aachen `__ (in German) diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index 97480db80..3bf2c475f 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -224,7 +224,7 @@ locks with the following command: d369ccc7d126594950bf74f0a348d5d98d9e99f3215082eb69bf02dc9b3e464c The ``find`` command searches for a given -`pattern `__ in the +`pattern `__ in the repository. .. code-block:: console From 5ac24a974465eb0339877e72164f1ffa349d14ae Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 26 Mar 2023 21:55:27 +0200 Subject: [PATCH 019/127] Switch back to monthly dependabot updates --- .github/dependabot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d608a8244..07f6b705b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,10 +4,10 @@ updates: - package-ecosystem: "gomod" directory: "/" # Location of package manifests schedule: - interval: "weekly" + interval: "monthly" # Dependencies listed in .github/workflows/*.yml - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "weekly" + interval: "monthly" From 220d9379751c215375f0b5b1d7f5193b339681fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 26 Mar 2023 20:13:34 +0000 Subject: [PATCH 020/127] build(deps): bump github.com/minio/minio-go/v7 from 7.0.49 to 7.0.50 Bumps [github.com/minio/minio-go/v7](https://github.com/minio/minio-go) from 7.0.49 to 7.0.50. - [Release notes](https://github.com/minio/minio-go/releases) - [Commits](https://github.com/minio/minio-go/compare/v7.0.49...v7.0.50) --- updated-dependencies: - dependency-name: github.com/minio/minio-go/v7 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 59acf89d0..af90bd48e 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/juju/ratelimit v1.0.2 github.com/klauspost/compress v1.16.0 github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5 - github.com/minio/minio-go/v7 v7.0.49 + github.com/minio/minio-go/v7 v7.0.50 github.com/minio/sha256-simd v1.0.0 github.com/ncw/swift/v2 v2.0.1 github.com/pkg/errors v0.9.1 @@ -52,7 +52,7 @@ require ( github.com/googleapis/gax-go/v2 v2.7.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.3 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/kr/fs v0.1.0 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect diff --git a/go.sum b/go.sum index 9bc87fc85..b85c3cb87 100644 --- a/go.sum +++ b/go.sum @@ -107,8 +107,8 @@ github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= -github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5 h1:OUlGa6AAolmjyPtILbMJ8vHayz5wd4wBUloheGcMhfA= @@ -116,8 +116,8 @@ github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5/go.mod h1:4FCXMUWo9 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.49 h1:dE5DfOtnXMXCjr/HWI6zN9vCrY6Sv666qhhiwUMvGV4= -github.com/minio/minio-go/v7 v7.0.49/go.mod h1:UI34MvQEiob3Cf/gGExGMmzugkM/tNgbFypNDy5LMVc= +github.com/minio/minio-go/v7 v7.0.50 h1:4IL4V8m/kI90ZL6GupCARZVrBv8/XrcKcJhaJ3iz68k= +github.com/minio/minio-go/v7 v7.0.50/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= From 21edbdc3ac9adc3666a356ba8c470805ed0c0f9e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 26 Mar 2023 20:14:09 +0000 Subject: [PATCH 021/127] build(deps): bump cloud.google.com/go/storage from 1.29.0 to 1.30.1 Bumps [cloud.google.com/go/storage](https://github.com/googleapis/google-cloud-go) from 1.29.0 to 1.30.1. - [Release notes](https://github.com/googleapis/google-cloud-go/releases) - [Changelog](https://github.com/googleapis/google-cloud-go/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-cloud-go/compare/pubsub/v1.29.0...spanner/v1.30.1) --- updated-dependencies: - dependency-name: cloud.google.com/go/storage dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 14 +++++++------- go.sum | 32 ++++++++++++++++---------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 59acf89d0..899dfe8ca 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/restic/restic require ( - cloud.google.com/go/storage v1.29.0 + cloud.google.com/go/storage v1.30.1 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 github.com/anacrolix/fuse v0.2.0 @@ -31,14 +31,14 @@ require ( golang.org/x/sys v0.6.0 golang.org/x/term v0.6.0 golang.org/x/text v0.8.0 - google.golang.org/api v0.111.0 + google.golang.org/api v0.114.0 ) require ( - cloud.google.com/go v0.108.0 // indirect + cloud.google.com/go v0.110.0 // indirect cloud.google.com/go/compute v1.18.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.11.0 // indirect + cloud.google.com/go/iam v0.12.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/dnaeon/go-vcr v1.2.0 // indirect @@ -49,7 +49,7 @@ require ( github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b // indirect github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.7.0 // indirect + github.com/googleapis/gax-go/v2 v2.7.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.3 // indirect @@ -63,9 +63,9 @@ require ( go.opencensus.io v0.24.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230223222841-637eb2293923 // indirect + google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 // indirect google.golang.org/grpc v1.53.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/protobuf v1.29.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 9bc87fc85..a8f045b07 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,15 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.108.0 h1:xntQwnfn8oHGX0crLVinvHM+AhXvi3QHQIEcX/2hiWk= -cloud.google.com/go v0.108.0/go.mod h1:lNUfQqusBJp0bgAg6qrHgYFYbTB+dOiob1itwnlD33Q= +cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/iam v0.11.0 h1:kwCWfKwB6ePZoZnGLwrd3B6Ru/agoHANTUBWpVNIdnM= -cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= -cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= -cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= -cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= +cloud.google.com/go/iam v0.12.0 h1:DRtTY29b75ciH6Ov1PHb4/iat2CLCvrOm40Q0a6DFpE= +cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= +cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= +cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.1 h1:gVXuXcWd1i4C2Ruxe321aU+IKGaStvGB/S90PUPB/W8= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.1/go.mod h1:DffdKW9RFqa5VgmsjUOsS7UE7eiA5iAvYUs63bhKQ0M= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= @@ -82,7 +82,7 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b h1:8htHrh2bw9c7Idkb7YNac+ZpTqLMjRpI+FWu51ltaQc= github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= @@ -91,8 +91,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= -github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= -github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= @@ -237,8 +237,8 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/api v0.111.0 h1:bwKi+z2BsdwYFRKrqwutM+axAlYLz83gt5pDSXCJT+0= -google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0= +google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= @@ -246,8 +246,8 @@ google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230223222841-637eb2293923 h1:znp6mq/drrY+6khTAlJUDNFFcDGV2ENLYKpMq8SyCds= -google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 h1:khxVcsk/FhnzxMKOyD+TDGwjbEOpcPuIpmafPGFmhMA= +google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= @@ -266,8 +266,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM= +google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= From 0ed5c20c57cf499c1225e8a25e2f5bae004d6ec9 Mon Sep 17 00:00:00 2001 From: Alex Duchesne Date: Tue, 28 Mar 2023 23:01:46 -0400 Subject: [PATCH 022/127] Added restic.exe to the .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 812d314b6..b7201c26b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /restic +/restic.exe /.vagrant /.vscode From f875a8843d876f8ae060edb8fd47f20baabd0fae Mon Sep 17 00:00:00 2001 From: Mark Herrmann Date: Fri, 28 Oct 2022 17:44:34 +0200 Subject: [PATCH 023/127] restore: Add progress bar Co-authored-by: Mark Herrmann --- changelog/unreleased/issue-3627 | 9 + cmd/restic/cmd_restore.go | 45 ++++- cmd/restic/integration_test.go | 8 +- internal/restorer/filerestorer.go | 14 +- internal/restorer/filerestorer_test.go | 6 +- internal/restorer/restorer.go | 23 ++- internal/restorer/restorer_test.go | 12 +- internal/restorer/restorer_unix_test.go | 2 +- internal/ui/restore/progressformatter.go | 131 ++++++++++++++ internal/ui/restore/progressformatter_test.go | 170 ++++++++++++++++++ 10 files changed, 399 insertions(+), 21 deletions(-) create mode 100644 changelog/unreleased/issue-3627 create mode 100644 internal/ui/restore/progressformatter.go create mode 100644 internal/ui/restore/progressformatter_test.go diff --git a/changelog/unreleased/issue-3627 b/changelog/unreleased/issue-3627 new file mode 100644 index 000000000..edbbdbb33 --- /dev/null +++ b/changelog/unreleased/issue-3627 @@ -0,0 +1,9 @@ +Enhancement: Show progress bar during restore + +The `restore` command now shows a progress report while restoring files. + +Example: [0:42] 5.76% 23 files 12.98 MiB, total 3456 files 23.54 GiB + +https://github.com/restic/restic/issues/3627 +https://github.com/restic/restic/pull/3991 +https://forum.restic.net/t/progress-bar-for-restore/5210 diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 579711662..a8b4f8069 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -3,6 +3,7 @@ package main import ( "context" "strings" + "sync" "time" "github.com/restic/restic/internal/debug" @@ -10,6 +11,9 @@ import ( "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restorer" + "github.com/restic/restic/internal/ui" + restoreui "github.com/restic/restic/internal/ui/restore" + "github.com/restic/restic/internal/ui/termstatus" "github.com/spf13/cobra" ) @@ -31,7 +35,31 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runRestore(cmd.Context(), restoreOptions, globalOptions, args) + ctx := cmd.Context() + var wg sync.WaitGroup + cancelCtx, cancel := context.WithCancel(ctx) + defer func() { + // shutdown termstatus + cancel() + wg.Wait() + }() + + term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet) + wg.Add(1) + go func() { + defer wg.Done() + term.Run(cancelCtx) + }() + + // allow usage of warnf / verbosef + prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr + defer func() { + globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr + }() + stdioWrapper := ui.NewStdioWrapper(term) + globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr() + + return runRestore(ctx, restoreOptions, globalOptions, term, args) }, } @@ -64,7 +92,9 @@ func init() { flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content") } -func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, args []string) error { +func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, + term *termstatus.Terminal, args []string) error { + hasExcludes := len(opts.Exclude) > 0 || len(opts.InsensitiveExclude) > 0 hasIncludes := len(opts.Include) > 0 || len(opts.InsensitiveInclude) > 0 @@ -145,7 +175,12 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, a return err } - res := restorer.NewRestorer(ctx, repo, sn, opts.Sparse) + var progress *restoreui.Progress + if !globalOptions.Quiet && !globalOptions.JSON { + progress = restoreui.NewProgress(restoreui.NewProgressPrinter(term), calculateProgressInterval(!gopts.Quiet, gopts.JSON)) + } + + res := restorer.NewRestorer(ctx, repo, sn, opts.Sparse, progress) totalErrors := 0 res.Error = func(location string, err error) error { @@ -209,6 +244,10 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, a return err } + if progress != nil { + progress.Finish() + } + if totalErrors > 0 { return errors.Fatalf("There were %d errors\n", totalErrors) } diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index c87722f02..d3882116e 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -112,7 +112,7 @@ func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths [ }, } - rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{"latest"})) + rtest.OK(t, runRestore(context.TODO(), opts, gopts, nil, []string{"latest"})) } func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, excludes []string) { @@ -121,7 +121,7 @@ func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snaps Exclude: excludes, } - rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{snapshotID.String()})) + rtest.OK(t, runRestore(context.TODO(), opts, gopts, nil, []string{snapshotID.String()})) } func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includes []string) { @@ -130,11 +130,11 @@ func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snaps Include: includes, } - rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{snapshotID.String()})) + rtest.OK(t, runRestore(context.TODO(), opts, gopts, nil, []string{snapshotID.String()})) } func testRunRestoreAssumeFailure(t testing.TB, snapshotID string, opts RestoreOptions, gopts GlobalOptions) error { - err := runRestore(context.TODO(), opts, gopts, []string{snapshotID}) + err := runRestore(context.TODO(), opts, gopts, nil, []string{snapshotID}) return err } diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index 2deef1cd2..75a19b4fb 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -12,6 +12,7 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/restore" ) // TODO if a blob is corrupt, there may be good blob copies in other packs @@ -54,6 +55,7 @@ type fileRestorer struct { filesWriter *filesWriter zeroChunk restic.ID sparse bool + progress *restore.Progress dst string files []*fileInfo @@ -65,7 +67,8 @@ func newFileRestorer(dst string, key *crypto.Key, idx func(restic.BlobHandle) []restic.PackedBlob, connections uint, - sparse bool) *fileRestorer { + sparse bool, + progress *restore.Progress) *fileRestorer { // as packs are streamed the concurrency is limited by IO workerCount := int(connections) @@ -77,6 +80,7 @@ func newFileRestorer(dst string, filesWriter: newFilesWriter(workerCount), zeroChunk: repository.ZeroChunk(), sparse: sparse, + progress: progress, workerCount: workerCount, dst: dst, Error: restorerAbortOnAllErrors, @@ -268,7 +272,13 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { file.inProgress = true createSize = file.size } - return r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize, file.sparse) + writeErr := r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize, file.sparse) + + if r.progress != nil { + r.progress.AddProgress(file.location, uint64(len(blobData)), uint64(file.size)) + } + + return writeErr } err := sanitizeError(file, writeToFile()) if err != nil { diff --git a/internal/restorer/filerestorer_test.go b/internal/restorer/filerestorer_test.go index b39afa249..e798f2b8b 100644 --- a/internal/restorer/filerestorer_test.go +++ b/internal/restorer/filerestorer_test.go @@ -150,7 +150,7 @@ func newTestRepo(content []TestFile) *TestRepo { func restoreAndVerify(t *testing.T, tempdir string, content []TestFile, files map[string]bool, sparse bool) { repo := newTestRepo(content) - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, sparse) + r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, sparse, nil) if files == nil { r.files = repo.files @@ -265,7 +265,7 @@ func TestErrorRestoreFiles(t *testing.T) { return loadError } - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false) + r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false, nil) r.files = repo.files err := r.restoreFiles(context.TODO()) @@ -304,7 +304,7 @@ func testPartialDownloadError(t *testing.T, part int) { return loader(ctx, h, length, offset, fn) } - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false) + r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false, nil) r.files = repo.files r.Error = func(s string, e error) error { // ignore errors as in the `restore` command diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 4dfe3c3a8..289883ed0 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -10,6 +10,7 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" + restoreui "github.com/restic/restic/internal/ui/restore" "golang.org/x/sync/errgroup" ) @@ -20,6 +21,8 @@ type Restorer struct { sn *restic.Snapshot sparse bool + progress *restoreui.Progress + Error func(location string, err error) error SelectFilter func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) } @@ -27,12 +30,14 @@ type Restorer struct { var restorerAbortOnAllErrors = func(location string, err error) error { return err } // NewRestorer creates a restorer preloaded with the content from the snapshot id. -func NewRestorer(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, sparse bool) *Restorer { +func NewRestorer(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, sparse bool, + progress *restoreui.Progress) *Restorer { r := &Restorer{ repo: repo, sparse: sparse, Error: restorerAbortOnAllErrors, SelectFilter: func(string, string, *restic.Node) (bool, bool) { return true, true }, + progress: progress, sn: sn, } @@ -186,6 +191,11 @@ func (res *Restorer) restoreHardlinkAt(node *restic.Node, target, path, location if err != nil { return errors.WithStack(err) } + + if res.progress != nil { + res.progress.AddProgress(location, 0, 0) + } + // TODO investigate if hardlinks have separate metadata on any supported system return res.restoreNodeMetadataTo(node, path, location) } @@ -200,6 +210,10 @@ func (res *Restorer) restoreEmptyFileAt(node *restic.Node, target, location stri return err } + if res.progress != nil { + res.progress.AddProgress(location, 0, 0) + } + return res.restoreNodeMetadataTo(node, target, location) } @@ -215,7 +229,8 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { } idx := NewHardlinkIndex() - filerestorer := newFileRestorer(dst, res.repo.Backend().Load, res.repo.Key(), res.repo.Index().Lookup, res.repo.Connections(), res.sparse) + filerestorer := newFileRestorer(dst, res.repo.Backend().Load, res.repo.Key(), res.repo.Index().Lookup, + res.repo.Connections(), res.sparse, res.progress) filerestorer.Error = res.Error debug.Log("first pass for %q", dst) @@ -242,6 +257,10 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { return nil } + if res.progress != nil { + res.progress.AddFile(node.Size) + } + if node.Size == 0 { return nil // deal with empty files later } diff --git a/internal/restorer/restorer_test.go b/internal/restorer/restorer_test.go index d6cd0c80a..1b0883bbb 100644 --- a/internal/restorer/restorer_test.go +++ b/internal/restorer/restorer_test.go @@ -325,7 +325,7 @@ func TestRestorer(t *testing.T) { sn, id := saveSnapshot(t, repo, test.Snapshot) t.Logf("snapshot saved as %v", id.Str()) - res := NewRestorer(context.TODO(), repo, sn, false) + res := NewRestorer(context.TODO(), repo, sn, false, nil) tempdir := rtest.TempDir(t) // make sure we're creating a new subdir of the tempdir @@ -442,7 +442,7 @@ func TestRestorerRelative(t *testing.T) { sn, id := saveSnapshot(t, repo, test.Snapshot) t.Logf("snapshot saved as %v", id.Str()) - res := NewRestorer(context.TODO(), repo, sn, false) + res := NewRestorer(context.TODO(), repo, sn, false, nil) tempdir := rtest.TempDir(t) cleanup := rtest.Chdir(t, tempdir) @@ -671,7 +671,7 @@ func TestRestorerTraverseTree(t *testing.T) { repo := repository.TestRepository(t) sn, _ := saveSnapshot(t, repo, test.Snapshot) - res := NewRestorer(context.TODO(), repo, sn, false) + res := NewRestorer(context.TODO(), repo, sn, false, nil) res.SelectFilter = test.Select @@ -747,7 +747,7 @@ func TestRestorerConsistentTimestampsAndPermissions(t *testing.T) { }, }) - res := NewRestorer(context.TODO(), repo, sn, false) + res := NewRestorer(context.TODO(), repo, sn, false, nil) res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { switch filepath.ToSlash(item) { @@ -802,7 +802,7 @@ func TestVerifyCancel(t *testing.T) { repo := repository.TestRepository(t) sn, _ := saveSnapshot(t, repo, snapshot) - res := NewRestorer(context.TODO(), repo, sn, false) + res := NewRestorer(context.TODO(), repo, sn, false, nil) tempdir := rtest.TempDir(t) ctx, cancel := context.WithCancel(context.Background()) @@ -844,7 +844,7 @@ func TestRestorerSparseFiles(t *testing.T) { archiver.SnapshotOptions{}) rtest.OK(t, err) - res := NewRestorer(context.TODO(), repo, sn, true) + res := NewRestorer(context.TODO(), repo, sn, true, nil) tempdir := rtest.TempDir(t) ctx, cancel := context.WithCancel(context.Background()) diff --git a/internal/restorer/restorer_unix_test.go b/internal/restorer/restorer_unix_test.go index dc327a9c9..e9c521e36 100644 --- a/internal/restorer/restorer_unix_test.go +++ b/internal/restorer/restorer_unix_test.go @@ -29,7 +29,7 @@ func TestRestorerRestoreEmptyHardlinkedFileds(t *testing.T) { }, }) - res := NewRestorer(context.TODO(), repo, sn, false) + res := NewRestorer(context.TODO(), repo, sn, false, nil) res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { return true, true diff --git a/internal/ui/restore/progressformatter.go b/internal/ui/restore/progressformatter.go new file mode 100644 index 000000000..a89cc628e --- /dev/null +++ b/internal/ui/restore/progressformatter.go @@ -0,0 +1,131 @@ +package restore + +import ( + "fmt" + "sync" + "time" + + "github.com/restic/restic/internal/ui" + "github.com/restic/restic/internal/ui/progress" +) + +type Progress struct { + updater progress.Updater + m sync.Mutex + + progressInfoMap map[string]progressInfoEntry + filesFinished uint64 + filesTotal uint64 + allBytesWritten uint64 + allBytesTotal uint64 + started time.Time + + printer ProgressPrinter +} + +type progressInfoEntry struct { + bytesWritten uint64 + bytesTotal uint64 +} + +type ProgressPrinter interface { + Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) + Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) +} + +func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress { + p := &Progress{ + progressInfoMap: make(map[string]progressInfoEntry), + started: time.Now(), + printer: printer, + } + p.updater = *progress.NewUpdater(interval, p.update) + return p +} + +func (p *Progress) update(runtime time.Duration, final bool) { + p.m.Lock() + defer p.m.Unlock() + + if !final { + p.printer.Update(p.filesFinished, p.filesTotal, p.allBytesWritten, p.allBytesTotal, runtime) + } else { + p.printer.Finish(p.filesFinished, p.filesTotal, p.allBytesWritten, p.allBytesTotal, runtime) + } +} + +// AddFile starts tracking a new file with the given size +func (p *Progress) AddFile(size uint64) { + p.m.Lock() + defer p.m.Unlock() + + p.filesTotal++ + p.allBytesTotal += size +} + +// AddProgress accumulates the number of bytes written for a file +func (p *Progress) AddProgress(name string, bytesWrittenPortion uint64, bytesTotal uint64) { + p.m.Lock() + defer p.m.Unlock() + + entry, exists := p.progressInfoMap[name] + if !exists { + entry.bytesTotal = bytesTotal + } + entry.bytesWritten += bytesWrittenPortion + p.progressInfoMap[name] = entry + + p.allBytesWritten += bytesWrittenPortion + if entry.bytesWritten == entry.bytesTotal { + delete(p.progressInfoMap, name) + p.filesFinished++ + } +} + +func (p *Progress) Finish() { + p.updater.Done() +} + +type term interface { + Print(line string) + SetStatus(lines []string) +} + +type textPrinter struct { + terminal term +} + +func NewProgressPrinter(terminal term) ProgressPrinter { + return &textPrinter{ + terminal: terminal, + } +} + +func (t *textPrinter) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { + timeLeft := ui.FormatDuration(duration) + formattedAllBytesWritten := ui.FormatBytes(allBytesWritten) + formattedAllBytesTotal := ui.FormatBytes(allBytesTotal) + allPercent := ui.FormatPercent(allBytesWritten, allBytesTotal) + progress := fmt.Sprintf("[%s] %s %v files %s, total %v files %v", + timeLeft, allPercent, filesFinished, formattedAllBytesWritten, filesTotal, formattedAllBytesTotal) + + t.terminal.SetStatus([]string{progress}) +} + +func (t *textPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { + t.terminal.SetStatus([]string{}) + + timeLeft := ui.FormatDuration(duration) + formattedAllBytesTotal := ui.FormatBytes(allBytesTotal) + + var summary string + if filesFinished == filesTotal && allBytesWritten == allBytesTotal { + summary = fmt.Sprintf("Summary: Restored %d Files (%s) in %s", filesTotal, formattedAllBytesTotal, timeLeft) + } else { + formattedAllBytesWritten := ui.FormatBytes(allBytesWritten) + summary = fmt.Sprintf("Summary: Restored %d / %d Files (%s / %s) in %s", + filesFinished, filesTotal, formattedAllBytesWritten, formattedAllBytesTotal, timeLeft) + } + + t.terminal.Print(summary) +} diff --git a/internal/ui/restore/progressformatter_test.go b/internal/ui/restore/progressformatter_test.go new file mode 100644 index 000000000..0cc4ea1ba --- /dev/null +++ b/internal/ui/restore/progressformatter_test.go @@ -0,0 +1,170 @@ +package restore + +import ( + "testing" + "time" + + "github.com/restic/restic/internal/test" +) + +type printerTraceEntry struct { + filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64 + + duration time.Duration + isFinished bool +} + +type printerTrace []printerTraceEntry + +type mockPrinter struct { + trace printerTrace +} + +const mockFinishDuration = 42 * time.Second + +func (p *mockPrinter) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { + p.trace = append(p.trace, printerTraceEntry{filesFinished, filesTotal, allBytesWritten, allBytesTotal, duration, false}) +} +func (p *mockPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { + p.trace = append(p.trace, printerTraceEntry{filesFinished, filesTotal, allBytesWritten, allBytesTotal, mockFinishDuration, true}) +} + +func testProgress(fn func(progress *Progress) bool) printerTrace { + printer := &mockPrinter{} + progress := NewProgress(printer, 0) + final := fn(progress) + progress.update(0, final) + trace := append(printerTrace{}, printer.trace...) + // cleanup to avoid goroutine leak, but copy trace first + progress.Finish() + return trace +} + +func TestNew(t *testing.T) { + result := testProgress(func(progress *Progress) bool { + return false + }) + test.Equals(t, printerTrace{ + printerTraceEntry{0, 0, 0, 0, 0, false}, + }, result) +} + +func TestAddFile(t *testing.T) { + fileSize := uint64(100) + + result := testProgress(func(progress *Progress) bool { + progress.AddFile(fileSize) + return false + }) + test.Equals(t, printerTrace{ + printerTraceEntry{0, 1, 0, fileSize, 0, false}, + }, result) +} + +func TestFirstProgressOnAFile(t *testing.T) { + expectedBytesWritten := uint64(5) + expectedBytesTotal := uint64(100) + + result := testProgress(func(progress *Progress) bool { + progress.AddFile(expectedBytesTotal) + progress.AddProgress("test", expectedBytesWritten, expectedBytesTotal) + return false + }) + test.Equals(t, printerTrace{ + printerTraceEntry{0, 1, expectedBytesWritten, expectedBytesTotal, 0, false}, + }, result) +} + +func TestLastProgressOnAFile(t *testing.T) { + fileSize := uint64(100) + + result := testProgress(func(progress *Progress) bool { + progress.AddFile(fileSize) + progress.AddProgress("test", 30, fileSize) + progress.AddProgress("test", 35, fileSize) + progress.AddProgress("test", 35, fileSize) + return false + }) + test.Equals(t, printerTrace{ + printerTraceEntry{1, 1, fileSize, fileSize, 0, false}, + }, result) +} + +func TestLastProgressOnLastFile(t *testing.T) { + fileSize := uint64(100) + + result := testProgress(func(progress *Progress) bool { + progress.AddFile(fileSize) + progress.AddFile(50) + progress.AddProgress("test1", 50, 50) + progress.AddProgress("test2", 50, fileSize) + progress.AddProgress("test2", 50, fileSize) + return false + }) + test.Equals(t, printerTrace{ + printerTraceEntry{2, 2, 50 + fileSize, 50 + fileSize, 0, false}, + }, result) +} + +func TestSummaryOnSuccess(t *testing.T) { + fileSize := uint64(100) + + result := testProgress(func(progress *Progress) bool { + progress.AddFile(fileSize) + progress.AddFile(50) + progress.AddProgress("test1", 50, 50) + progress.AddProgress("test2", fileSize, fileSize) + return true + }) + test.Equals(t, printerTrace{ + printerTraceEntry{2, 2, 50 + fileSize, 50 + fileSize, mockFinishDuration, true}, + }, result) +} + +func TestSummaryOnErrors(t *testing.T) { + fileSize := uint64(100) + + result := testProgress(func(progress *Progress) bool { + progress.AddFile(fileSize) + progress.AddFile(50) + progress.AddProgress("test1", 50, 50) + progress.AddProgress("test2", fileSize/2, fileSize) + return true + }) + test.Equals(t, printerTrace{ + printerTraceEntry{1, 2, 50 + fileSize/2, 50 + fileSize, mockFinishDuration, true}, + }, result) +} + +type mockTerm struct { + output []string +} + +func (m *mockTerm) Print(line string) { + m.output = append(m.output, line) +} + +func (m *mockTerm) SetStatus(lines []string) { + m.output = append([]string{}, lines...) +} + +func TestPrintUpdate(t *testing.T) { + term := &mockTerm{} + printer := NewProgressPrinter(term) + printer.Update(3, 11, 29, 47, 5*time.Second) + test.Equals(t, []string{"[0:05] 61.70% 3 files 29 B, total 11 files 47 B"}, term.output) +} + +func TestPrintSummaryOnSuccess(t *testing.T) { + term := &mockTerm{} + printer := NewProgressPrinter(term) + printer.Finish(11, 11, 47, 47, 5*time.Second) + test.Equals(t, []string{"Summary: Restored 11 Files (47 B) in 0:05"}, term.output) +} + +func TestPrintSummaryOnErrors(t *testing.T) { + term := &mockTerm{} + printer := NewProgressPrinter(term) + printer.Finish(3, 11, 29, 47, 5*time.Second) + test.Equals(t, []string{"Summary: Restored 3 / 11 Files (29 B / 47 B) in 0:05"}, term.output) +} From cacc48fc099bb8be9590d1113a673537677ce1a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Apr 2023 10:27:30 +0000 Subject: [PATCH 024/127] build(deps): bump github.com/Azure/azure-sdk-for-go/sdk/azcore Bumps [github.com/Azure/azure-sdk-for-go/sdk/azcore](https://github.com/Azure/azure-sdk-for-go) from 1.3.1 to 1.4.0. - [Release notes](https://github.com/Azure/azure-sdk-for-go/releases) - [Changelog](https://github.com/Azure/azure-sdk-for-go/blob/main/documentation/release.md) - [Commits](https://github.com/Azure/azure-sdk-for-go/compare/sdk/azcore/v1.3.1...sdk/azcore/v1.4.0) --- updated-dependencies: - dependency-name: github.com/Azure/azure-sdk-for-go/sdk/azcore dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 899dfe8ca..0e41c4c4e 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/restic/restic require ( cloud.google.com/go/storage v1.30.1 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.1 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 github.com/anacrolix/fuse v0.2.0 github.com/cenkalti/backoff/v4 v4.2.0 diff --git a/go.sum b/go.sum index a8f045b07..3bed20517 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.1 h1:gVXuXcWd1i4C2Ruxe321aU+IKGaStvGB/S90PUPB/W8= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.1/go.mod h1:DffdKW9RFqa5VgmsjUOsS7UE7eiA5iAvYUs63bhKQ0M= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJVR2cMC8lvZcimipiEY= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= From 9584cbda90a0a6fb9b11d57fdc48a49d52660ad5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Apr 2023 10:27:38 +0000 Subject: [PATCH 025/127] build(deps): bump google.golang.org/api from 0.114.0 to 0.116.0 Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.114.0 to 0.116.0. - [Release notes](https://github.com/googleapis/google-api-go-client/releases) - [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.114.0...v0.116.0) --- updated-dependencies: - dependency-name: google.golang.org/api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 16 ++++++++-------- go.sum | 32 ++++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 899dfe8ca..89c25f5ed 100644 --- a/go.mod +++ b/go.mod @@ -31,25 +31,25 @@ require ( golang.org/x/sys v0.6.0 golang.org/x/term v0.6.0 golang.org/x/text v0.8.0 - google.golang.org/api v0.114.0 + google.golang.org/api v0.116.0 ) require ( cloud.google.com/go v0.110.0 // indirect - cloud.google.com/go/compute v1.18.0 // indirect + cloud.google.com/go/compute v1.19.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.12.0 // indirect + cloud.google.com/go/iam v0.13.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/dnaeon/go-vcr v1.2.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/fgprof v0.9.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b // indirect github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.7.1 // indirect + github.com/googleapis/gax-go/v2 v2.8.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.3 // indirect @@ -63,9 +63,9 @@ require ( go.opencensus.io v0.24.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 // indirect - google.golang.org/grpc v1.53.0 // indirect - google.golang.org/protobuf v1.29.1 // indirect + google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 // indirect + google.golang.org/grpc v1.54.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index a8f045b07..196afad54 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,12 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= -cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= -cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.0 h1:+9zda3WGgW1ZSTlVppLCYFIr48Pa35q1uG2N1itbCEQ= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/iam v0.12.0 h1:DRtTY29b75ciH6Ov1PHb4/iat2CLCvrOm40Q0a6DFpE= -cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= @@ -70,8 +70,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -91,8 +91,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= -github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= -github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.8.0 h1:UBtEZqx1bjXtOQ5BVTkuYghXrr3N4V123VKJK67vJZc= +github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= @@ -237,8 +237,8 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= -google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/api v0.116.0 h1:09tOPVufPwfm5W4aA8EizGHJ7BcoRDsIareM2a15gO4= +google.golang.org/api v0.116.0/go.mod h1:9cD4/t6uvd9naoEJFA+M96d0IuB6BqFuyhpw68+mRGg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= @@ -246,15 +246,15 @@ google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 h1:khxVcsk/FhnzxMKOyD+TDGwjbEOpcPuIpmafPGFmhMA= -google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 h1:0BOZf6qNozI3pkN3fJLwNubheHJYHhMh91GRFOWWK08= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -266,8 +266,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM= -google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= From 9358a5fb3770e754319dd2467d75425af8fc15af Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Apr 2023 11:03:25 +0000 Subject: [PATCH 026/127] build(deps): bump github.com/Azure/azure-sdk-for-go/sdk/storage/azblob Bumps [github.com/Azure/azure-sdk-for-go/sdk/storage/azblob](https://github.com/Azure/azure-sdk-for-go) from 0.5.1 to 1.0.0. - [Release notes](https://github.com/Azure/azure-sdk-for-go/releases) - [Changelog](https://github.com/Azure/azure-sdk-for-go/blob/main/documentation/release.md) - [Commits](https://github.com/Azure/azure-sdk-for-go/compare/sdk/armcore/v0.5.1...v1.0) --- updated-dependencies: - dependency-name: github.com/Azure/azure-sdk-for-go/sdk/storage/azblob dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d0e514220..752e88019 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/restic/restic require ( cloud.google.com/go/storage v1.30.1 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 github.com/anacrolix/fuse v0.2.0 github.com/cenkalti/backoff/v4 v4.2.0 github.com/cespare/xxhash/v2 v2.2.0 diff --git a/go.sum b/go.sum index ce664c15a..493f95aad 100644 --- a/go.sum +++ b/go.sum @@ -15,8 +15,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDr github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= From faa83db9e477324baf90f965c771e92c0d8d76de Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 7 Apr 2023 15:05:55 +0200 Subject: [PATCH 027/127] azure: Adapt code to API change --- internal/backend/azure/azure.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index 02433795b..c92fa3f89 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -228,7 +228,7 @@ func (be *Backend) saveSmall(ctx context.Context, objName string, rd restic.Rewi reader := bytes.NewReader(buf) _, err = blockBlobClient.StageBlock(ctx, id, streaming.NopCloser(reader), &blockblob.StageBlockOptions{ - TransactionalContentMD5: rd.Hash(), + TransactionalValidation: blob.TransferValidationTypeMD5(rd.Hash()), }) if err != nil { return errors.Wrap(err, "StageBlock") @@ -271,7 +271,7 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi reader := bytes.NewReader(buf) debug.Log("StageBlock %v with %d bytes", id, len(buf)) _, err = blockBlobClient.StageBlock(ctx, id, streaming.NopCloser(reader), &blockblob.StageBlockOptions{ - TransactionalContentMD5: h[:], + TransactionalValidation: blob.TransferValidationTypeMD5(h[:]), }) if err != nil { From 806a0cdce3ee2542caa16dbcbf5bf279988e58b3 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 7 Apr 2023 15:31:20 +0200 Subject: [PATCH 028/127] bump minimum go version on Solaris to 1.20 --- build.go | 6 ++++++ changelog/unreleased/pull-4201 | 9 +++++++++ doc/020_installation.rst | 3 ++- 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/pull-4201 diff --git a/build.go b/build.go index dddc3b964..b3b7f5eee 100644 --- a/build.go +++ b/build.go @@ -380,6 +380,12 @@ func main() { } } + solarisMinVersion := GoVersion{Major: 1, Minor: 20, Patch: 0} + if env["GOARCH"] == "solaris" && !goVersion.AtLeast(solarisMinVersion) { + fmt.Fprintf(os.Stderr, "Detected version %s is too old, restic requires at least %s for Solaris\n", goVersion, solarisMinVersion) + os.Exit(1) + } + verbosePrintf("detected Go version %v\n", goVersion) preserveSymbols := false diff --git a/changelog/unreleased/pull-4201 b/changelog/unreleased/pull-4201 new file mode 100644 index 000000000..500bbdbb1 --- /dev/null +++ b/changelog/unreleased/pull-4201 @@ -0,0 +1,9 @@ +Change: Require Go 1.20 for Solaris builds + +Building restic on Solaris now requires Go 1.20, as the library used to access +Azure uses the mmap syscall, which is only available on Solaris starting from +Go 1.20. + +All other platforms continue to build with Go 1.18. + +https://github.com/restic/restic/pull/4201 diff --git a/doc/020_installation.rst b/doc/020_installation.rst index 4488e31f9..4d591356d 100644 --- a/doc/020_installation.rst +++ b/doc/020_installation.rst @@ -269,7 +269,8 @@ From Source *********** restic is written in the Go programming language and you need at least -Go version 1.18. Building restic may also work with older versions of Go, +Go version 1.18. Building for Solaris requires at least Go version 1.20. +Building restic may also work with older versions of Go, but that's not supported. See the `Getting started `__ guide of the Go project for instructions how to install Go. From ea59896bd61990cfa8b7b31b1c9eb9ebd7000ed4 Mon Sep 17 00:00:00 2001 From: jo Date: Thu, 16 Feb 2023 16:58:36 +0100 Subject: [PATCH 029/127] Add a global option --retry-lock Fixes restic#719 If the option is passed, restic will wait the specified duration of time and retry locking the repo every 10 seconds (or more often if the total timeout is relatively small). - Play nice with json output - Reduce wait time in lock tests - Rework timeout last attempt - Reduce test wait time to 0.1s - Use exponential back off for the retry lock - Don't pass gopts to lockRepo functions - Use global variable for retry sleep setup - Exit retry lock on cancel - Better wording for flag help - Reorder debug statement - Refactor tests - Lower max sleep time to 1m - Test that we cancel/timeout in time - Use non blocking sleep function - Refactor into minDuration func Co-authored-by: Julian Brost --- changelog/unreleased/issue-719 | 8 +++ cmd/restic/cmd_backup.go | 2 +- cmd/restic/cmd_cat.go | 2 +- cmd/restic/cmd_check.go | 2 +- cmd/restic/cmd_copy.go | 4 +- cmd/restic/cmd_debug.go | 4 +- cmd/restic/cmd_diff.go | 2 +- cmd/restic/cmd_dump.go | 2 +- cmd/restic/cmd_find.go | 2 +- cmd/restic/cmd_forget.go | 2 +- cmd/restic/cmd_key.go | 8 +-- cmd/restic/cmd_list.go | 8 +-- cmd/restic/cmd_migrate.go | 2 +- cmd/restic/cmd_mount.go | 2 +- cmd/restic/cmd_prune.go | 2 +- cmd/restic/cmd_rebuild_index.go | 2 +- cmd/restic/cmd_recover.go | 2 +- cmd/restic/cmd_restore.go | 2 +- cmd/restic/cmd_rewrite.go | 4 +- cmd/restic/cmd_snapshots.go | 2 +- cmd/restic/cmd_stats.go | 2 +- cmd/restic/cmd_tag.go | 2 +- cmd/restic/global.go | 2 + cmd/restic/lock.go | 61 +++++++++++++++++-- cmd/restic/lock_test.go | 101 +++++++++++++++++++++++++++----- doc/design.rst | 5 +- doc/manual_rest.rst | 2 + 27 files changed, 188 insertions(+), 51 deletions(-) create mode 100644 changelog/unreleased/issue-719 diff --git a/changelog/unreleased/issue-719 b/changelog/unreleased/issue-719 new file mode 100644 index 000000000..4f28ea83c --- /dev/null +++ b/changelog/unreleased/issue-719 @@ -0,0 +1,8 @@ +Enhancement: Add --retry-lock option + +This option allows to specify a duration for which restic will wait if there +already exists a conflicting lock within the repository. + +https://github.com/restic/restic/issues/719 +https://github.com/restic/restic/pull/2214 +https://github.com/restic/restic/pull/4107 diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 1244e2ed1..fcaff304d 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -506,7 +506,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter if !gopts.JSON { progressPrinter.V("lock repository") } - lock, ctx, err := lockRepo(ctx, repo) + lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index e7253e5b6..771731a58 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -45,7 +45,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { if !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index d56f7d0c9..e5f29a7e5 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -211,7 +211,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args if !gopts.NoLock { Verbosef("create exclusive lock for repository\n") var lock *restic.Lock - lock, ctx, err = lockRepoExclusive(ctx, repo) + lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 2f095972a..13767d98a 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -74,14 +74,14 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] if !gopts.NoLock { var srcLock *restic.Lock - srcLock, ctx, err = lockRepo(ctx, srcRepo) + srcLock, ctx, err = lockRepo(ctx, srcRepo, gopts.RetryLock, gopts.JSON) defer unlockRepo(srcLock) if err != nil { return err } } - dstLock, ctx, err := lockRepo(ctx, dstRepo) + dstLock, ctx, err := lockRepo(ctx, dstRepo, gopts.RetryLock, gopts.JSON) defer unlockRepo(dstLock) if err != nil { return err diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index c8626d46c..deade6d22 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -156,7 +156,7 @@ func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error if !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err @@ -462,7 +462,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er if !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index 0000fd18a..0861a7103 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -334,7 +334,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args [] if !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index cda7b65b9..34313f582 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -132,7 +132,7 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] if !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index e5457c3be..3ef5f26bf 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -575,7 +575,7 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args [] if !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index fbe4c1c8a..b958a81db 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -116,7 +116,7 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg if !opts.DryRun || !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepoExclusive(ctx, repo) + lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index 88b6d5c0c..62521d762 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -212,7 +212,7 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error { switch args[0] { case "list": - lock, ctx, err := lockRepo(ctx, repo) + lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err @@ -220,7 +220,7 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error { return listKeys(ctx, repo, gopts) case "add": - lock, ctx, err := lockRepo(ctx, repo) + lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err @@ -228,7 +228,7 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error { return addKey(ctx, repo, gopts) case "remove": - lock, ctx, err := lockRepoExclusive(ctx, repo) + lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err @@ -241,7 +241,7 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error { return deleteKey(ctx, repo, id) case "passwd": - lock, ctx, err := lockRepoExclusive(ctx, repo) + lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go index 4809092c0..bd02cedc7 100644 --- a/cmd/restic/cmd_list.go +++ b/cmd/restic/cmd_list.go @@ -31,19 +31,19 @@ func init() { cmdRoot.AddCommand(cmdList) } -func runList(ctx context.Context, cmd *cobra.Command, opts GlobalOptions, args []string) error { +func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args []string) error { if len(args) != 1 { return errors.Fatal("type not specified, usage: " + cmd.Use) } - repo, err := OpenRepository(ctx, opts) + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } - if !opts.NoLock && args[0] != "locks" { + if !gopts.NoLock && args[0] != "locks" { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_migrate.go b/cmd/restic/cmd_migrate.go index 6d614be39..fd2e762c0 100644 --- a/cmd/restic/cmd_migrate.go +++ b/cmd/restic/cmd_migrate.go @@ -122,7 +122,7 @@ func runMigrate(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, a return err } - lock, ctx, err := lockRepoExclusive(ctx, repo) + lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index 0501bfe89..ec3662d5c 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -123,7 +123,7 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args if !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index f59be2967..6104002b0 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -167,7 +167,7 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error opts.unsafeRecovery = true } - lock, ctx, err := lockRepoExclusive(ctx, repo) + lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_rebuild_index.go b/cmd/restic/cmd_rebuild_index.go index 6d49cb917..5d70a9e12 100644 --- a/cmd/restic/cmd_rebuild_index.go +++ b/cmd/restic/cmd_rebuild_index.go @@ -49,7 +49,7 @@ func runRebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts Global return err } - lock, ctx, err := lockRepoExclusive(ctx, repo) + lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index 65f4c8750..85dcc23d7 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -46,7 +46,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error { return err } - lock, ctx, err := lockRepo(ctx, repo) + lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index a8b4f8069..a3e602c8f 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -154,7 +154,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, if !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index 0d9aa1c8c..8a1b860ed 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -164,9 +164,9 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a var err error if opts.Forget { Verbosef("create exclusive lock for repository\n") - lock, ctx, err = lockRepoExclusive(ctx, repo) + lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) } else { - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) } defer unlockRepo(lock) if err != nil { diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index 2de8801cb..ba3644ee7 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -65,7 +65,7 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions if !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 55ba6f254..e8558d290 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -83,7 +83,7 @@ func runStats(ctx context.Context, gopts GlobalOptions, args []string) error { if !gopts.NoLock { var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo) + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_tag.go b/cmd/restic/cmd_tag.go index e5948ea02..fe4638547 100644 --- a/cmd/restic/cmd_tag.go +++ b/cmd/restic/cmd_tag.go @@ -111,7 +111,7 @@ func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []st if !gopts.NoLock { Verbosef("create exclusive lock for repository\n") var lock *restic.Lock - lock, ctx, err = lockRepoExclusive(ctx, repo) + lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/global.go b/cmd/restic/global.go index c2c288421..32f18a67f 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -59,6 +59,7 @@ type GlobalOptions struct { Quiet bool Verbose int NoLock bool + RetryLock time.Duration JSON bool CacheDir string NoCache bool @@ -115,6 +116,7 @@ func init() { // use empty paremeter name as `-v, --verbose n` instead of the correct `--verbose=n` is confusing f.CountVarP(&globalOptions.Verbose, "verbose", "v", "be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2)") f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repository, this allows some operations on read-only repositories") + f.DurationVar(&globalOptions.RetryLock, "retry-lock", 0, "retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries)") f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it") f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache `directory`. (default: use system default cache directory)") f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache") diff --git a/cmd/restic/lock.go b/cmd/restic/lock.go index f39a08db6..450922704 100644 --- a/cmd/restic/lock.go +++ b/cmd/restic/lock.go @@ -21,17 +21,29 @@ var globalLocks struct { sync.Once } -func lockRepo(ctx context.Context, repo restic.Repository) (*restic.Lock, context.Context, error) { - return lockRepository(ctx, repo, false) +func lockRepo(ctx context.Context, repo restic.Repository, retryLock time.Duration, json bool) (*restic.Lock, context.Context, error) { + return lockRepository(ctx, repo, false, retryLock, json) } -func lockRepoExclusive(ctx context.Context, repo restic.Repository) (*restic.Lock, context.Context, error) { - return lockRepository(ctx, repo, true) +func lockRepoExclusive(ctx context.Context, repo restic.Repository, retryLock time.Duration, json bool) (*restic.Lock, context.Context, error) { + return lockRepository(ctx, repo, true, retryLock, json) +} + +var ( + retrySleepStart = 5 * time.Second + retrySleepMax = 60 * time.Second +) + +func minDuration(a, b time.Duration) time.Duration { + if a <= b { + return a + } + return b } // lockRepository wraps the ctx such that it is cancelled when the repository is unlocked // cancelling the original context also stops the lock refresh -func lockRepository(ctx context.Context, repo restic.Repository, exclusive bool) (*restic.Lock, context.Context, error) { +func lockRepository(ctx context.Context, repo restic.Repository, exclusive bool, retryLock time.Duration, json bool) (*restic.Lock, context.Context, error) { // make sure that a repository is unlocked properly and after cancel() was // called by the cleanup handler in global.go globalLocks.Do(func() { @@ -43,7 +55,44 @@ func lockRepository(ctx context.Context, repo restic.Repository, exclusive bool) lockFn = restic.NewExclusiveLock } - lock, err := lockFn(ctx, repo) + var lock *restic.Lock + var err error + + retrySleep := minDuration(retrySleepStart, retryLock) + retryMessagePrinted := false + retryTimeout := time.After(retryLock) + +retryLoop: + for { + lock, err = lockFn(ctx, repo) + if err != nil && restic.IsAlreadyLocked(err) { + + if !retryMessagePrinted { + if !json { + Verbosef("repo already locked, waiting up to %s for the lock\n", retryLock) + } + retryMessagePrinted = true + } + + debug.Log("repo already locked, retrying in %v", retrySleep) + retrySleepCh := time.After(retrySleep) + + select { + case <-ctx.Done(): + return nil, ctx, ctx.Err() + case <-retryTimeout: + debug.Log("repo already locked, timeout expired") + // Last lock attempt + lock, err = lockFn(ctx, repo) + break retryLoop + case <-retrySleepCh: + retrySleep = minDuration(retrySleep*2, retrySleepMax) + } + } else { + // anything else, either a successful lock or another error + break retryLoop + } + } if restic.IsInvalidLock(err) { return nil, ctx, errors.Fatalf("%v\n\nthe `unlock --remove-all` command can be used to remove invalid locks. Make sure that no other restic process is accessing the repository when running the command", err) } diff --git a/cmd/restic/lock_test.go b/cmd/restic/lock_test.go index c074f15a6..ad9161739 100644 --- a/cmd/restic/lock_test.go +++ b/cmd/restic/lock_test.go @@ -3,11 +3,13 @@ package main import ( "context" "fmt" + "strings" "testing" "time" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test" ) @@ -23,8 +25,8 @@ func openTestRepo(t *testing.T, wrapper backendWrapper) (*repository.Repository, return repo, cleanup, env } -func checkedLockRepo(ctx context.Context, t *testing.T, repo restic.Repository) (*restic.Lock, context.Context) { - lock, wrappedCtx, err := lockRepo(ctx, repo) +func checkedLockRepo(ctx context.Context, t *testing.T, repo restic.Repository, env *testEnvironment) (*restic.Lock, context.Context) { + lock, wrappedCtx, err := lockRepo(ctx, repo, env.gopts.RetryLock, env.gopts.JSON) rtest.OK(t, err) rtest.OK(t, wrappedCtx.Err()) if lock.Stale() { @@ -34,10 +36,10 @@ func checkedLockRepo(ctx context.Context, t *testing.T, repo restic.Repository) } func TestLock(t *testing.T) { - repo, cleanup, _ := openTestRepo(t, nil) + repo, cleanup, env := openTestRepo(t, nil) defer cleanup() - lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo) + lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, env) unlockRepo(lock) if wrappedCtx.Err() == nil { t.Fatal("unlock did not cancel context") @@ -45,12 +47,12 @@ func TestLock(t *testing.T) { } func TestLockCancel(t *testing.T) { - repo, cleanup, _ := openTestRepo(t, nil) + repo, cleanup, env := openTestRepo(t, nil) defer cleanup() ctx, cancel := context.WithCancel(context.Background()) defer cancel() - lock, wrappedCtx := checkedLockRepo(ctx, t, repo) + lock, wrappedCtx := checkedLockRepo(ctx, t, repo, env) cancel() if wrappedCtx.Err() == nil { t.Fatal("canceled parent context did not cancel context") @@ -61,10 +63,10 @@ func TestLockCancel(t *testing.T) { } func TestLockUnlockAll(t *testing.T) { - repo, cleanup, _ := openTestRepo(t, nil) + repo, cleanup, env := openTestRepo(t, nil) defer cleanup() - lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo) + lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, env) _, err := unlockAll(0) rtest.OK(t, err) if wrappedCtx.Err() == nil { @@ -81,10 +83,10 @@ func TestLockConflict(t *testing.T) { repo2, err := OpenRepository(context.TODO(), env.gopts) rtest.OK(t, err) - lock, _, err := lockRepoExclusive(context.Background(), repo) + lock, _, err := lockRepoExclusive(context.Background(), repo, env.gopts.RetryLock, env.gopts.JSON) rtest.OK(t, err) defer unlockRepo(lock) - _, _, err = lockRepo(context.Background(), repo2) + _, _, err = lockRepo(context.Background(), repo2, env.gopts.RetryLock, env.gopts.JSON) if err == nil { t.Fatal("second lock should have failed") } @@ -104,7 +106,7 @@ func (b *writeOnceBackend) Save(ctx context.Context, h restic.Handle, rd restic. } func TestLockFailedRefresh(t *testing.T) { - repo, cleanup, _ := openTestRepo(t, func(r restic.Backend) (restic.Backend, error) { + repo, cleanup, env := openTestRepo(t, func(r restic.Backend) (restic.Backend, error) { return &writeOnceBackend{Backend: r}, nil }) defer cleanup() @@ -117,7 +119,7 @@ func TestLockFailedRefresh(t *testing.T) { refreshInterval, refreshabilityTimeout = ri, rt }() - lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo) + lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, env) select { case <-wrappedCtx.Done(): @@ -140,7 +142,7 @@ func (b *loggingBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re } func TestLockSuccessfulRefresh(t *testing.T) { - repo, cleanup, _ := openTestRepo(t, func(r restic.Backend) (restic.Backend, error) { + repo, cleanup, env := openTestRepo(t, func(r restic.Backend) (restic.Backend, error) { return &loggingBackend{ Backend: r, t: t, @@ -157,7 +159,7 @@ func TestLockSuccessfulRefresh(t *testing.T) { refreshInterval, refreshabilityTimeout = ri, rt }() - lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo) + lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, env) select { case <-wrappedCtx.Done(): @@ -168,3 +170,74 @@ func TestLockSuccessfulRefresh(t *testing.T) { // unlockRepo should not crash unlockRepo(lock) } + +func TestLockWaitTimeout(t *testing.T) { + repo, cleanup, env := openTestRepo(t, nil) + defer cleanup() + + elock, _, err := lockRepoExclusive(context.TODO(), repo, env.gopts.RetryLock, env.gopts.JSON) + test.OK(t, err) + + retryLock := 100 * time.Millisecond + + start := time.Now() + lock, _, err := lockRepo(context.TODO(), repo, retryLock, env.gopts.JSON) + duration := time.Since(start) + + test.Assert(t, err != nil, + "create normal lock with exclusively locked repo didn't return an error") + test.Assert(t, strings.Contains(err.Error(), "repository is already locked exclusively"), + "create normal lock with exclusively locked repo didn't return the correct error") + test.Assert(t, retryLock <= duration && duration < retryLock+5*time.Millisecond, + "create normal lock with exclusively locked repo didn't wait for the specified timeout") + + test.OK(t, lock.Unlock()) + test.OK(t, elock.Unlock()) +} +func TestLockWaitCancel(t *testing.T) { + repo, cleanup, env := openTestRepo(t, nil) + defer cleanup() + + elock, _, err := lockRepoExclusive(context.TODO(), repo, env.gopts.RetryLock, env.gopts.JSON) + test.OK(t, err) + + retryLock := 100 * time.Millisecond + cancelAfter := 40 * time.Millisecond + + ctx, cancel := context.WithCancel(context.TODO()) + time.AfterFunc(cancelAfter, cancel) + + start := time.Now() + lock, _, err := lockRepo(ctx, repo, retryLock, env.gopts.JSON) + duration := time.Since(start) + + test.Assert(t, err != nil, + "create normal lock with exclusively locked repo didn't return an error") + test.Assert(t, strings.Contains(err.Error(), "context canceled"), + "create normal lock with exclusively locked repo didn't return the correct error") + test.Assert(t, cancelAfter <= duration && duration < cancelAfter+5*time.Millisecond, + "create normal lock with exclusively locked repo didn't return in time") + + test.OK(t, lock.Unlock()) + test.OK(t, elock.Unlock()) +} + +func TestLockWaitSuccess(t *testing.T) { + repo, cleanup, env := openTestRepo(t, nil) + defer cleanup() + + elock, _, err := lockRepoExclusive(context.TODO(), repo, env.gopts.RetryLock, env.gopts.JSON) + test.OK(t, err) + + retryLock := 100 * time.Millisecond + unlockAfter := 40 * time.Millisecond + + time.AfterFunc(unlockAfter, func() { + test.OK(t, elock.Unlock()) + }) + + lock, _, err := lockRepo(context.TODO(), repo, retryLock, env.gopts.JSON) + test.OK(t, err) + + test.OK(t, lock.Unlock()) +} diff --git a/doc/design.rst b/doc/design.rst index 7102585ac..94dabdc34 100644 --- a/doc/design.rst +++ b/doc/design.rst @@ -603,7 +603,10 @@ that the process is dead and considers the lock to be stale. When a new lock is to be created and no other conflicting locks are detected, restic creates a new lock, waits, and checks if other locks appeared in the repository. Depending on the type of the other locks and -the lock to be created, restic either continues or fails. +the lock to be created, restic either continues or fails. If the +``--retry-lock`` option is specified, restic will retry +creating the lock periodically until it succeeds or the specified +timeout expires. Read and Write Ordering ======================= diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index 3bf2c475f..7f812f4e0 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -66,6 +66,7 @@ Usage help is available: -q, --quiet do not output comprehensive progress report -r, --repo repository repository to backup to or restore from (default: $RESTIC_REPOSITORY) --repository-file file file to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + --retry-lock duration retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) --tls-client-cert file path to a file containing PEM encoded TLS client certificate and private key -v, --verbose be verbose (specify multiple times or a level using --verbose=n, max level/times is 2) @@ -141,6 +142,7 @@ command: -q, --quiet do not output comprehensive progress report -r, --repo repository repository to backup to or restore from (default: $RESTIC_REPOSITORY) --repository-file file file to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + --retry-lock duration retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) --tls-client-cert file path to a file containing PEM encoded TLS client certificate and private key -v, --verbose be verbose (specify multiple times or a level using --verbose=n, max level/times is 2) From 64233ca0a7880c47c19b85eb64691ede3550153a Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 7 Apr 2023 19:43:00 +0200 Subject: [PATCH 030/127] lock: Improve debug logging in the test --- cmd/restic/lock_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/restic/lock_test.go b/cmd/restic/lock_test.go index c074f15a6..242252f42 100644 --- a/cmd/restic/lock_test.go +++ b/cmd/restic/lock_test.go @@ -136,7 +136,9 @@ type loggingBackend struct { func (b *loggingBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { b.t.Logf("save %v @ %v", h, time.Now()) - return b.Backend.Save(ctx, h, rd) + err := b.Backend.Save(ctx, h, rd) + b.t.Logf("save finished %v @ %v", h, time.Now()) + return err } func TestLockSuccessfulRefresh(t *testing.T) { @@ -161,7 +163,8 @@ func TestLockSuccessfulRefresh(t *testing.T) { select { case <-wrappedCtx.Done(): - t.Fatal("lock refresh failed") + // don't call t.Fatal to allow the lock to be properly cleaned up + t.Error("lock refresh failed", time.Now()) case <-time.After(2 * refreshabilityTimeout): // expected lock refresh to work } From cf1cc1fb72b11343915d7a900fd0b43ea5b3a1e8 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 7 Apr 2023 19:51:31 +0200 Subject: [PATCH 031/127] lock: Print stacktrace if TestLockSuccessfulRefresh fails --- cmd/restic/lock_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmd/restic/lock_test.go b/cmd/restic/lock_test.go index 242252f42..fd3af0724 100644 --- a/cmd/restic/lock_test.go +++ b/cmd/restic/lock_test.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "runtime" "testing" "time" @@ -165,6 +166,13 @@ func TestLockSuccessfulRefresh(t *testing.T) { case <-wrappedCtx.Done(): // don't call t.Fatal to allow the lock to be properly cleaned up t.Error("lock refresh failed", time.Now()) + + // Dump full stacktrace + buf := make([]byte, 1024*1024) + n := runtime.Stack(buf, true) + buf = buf[:n] + t.Log(string(buf)) + case <-time.After(2 * refreshabilityTimeout): // expected lock refresh to work } From 8ce5f2975882029caece81bc3bc25788e0b60cb5 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 7 Apr 2023 20:03:51 +0200 Subject: [PATCH 032/127] lock: increase test timeout tolerances to avoid test failures --- cmd/restic/lock_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/restic/lock_test.go b/cmd/restic/lock_test.go index ad9161739..cb386312c 100644 --- a/cmd/restic/lock_test.go +++ b/cmd/restic/lock_test.go @@ -188,7 +188,7 @@ func TestLockWaitTimeout(t *testing.T) { "create normal lock with exclusively locked repo didn't return an error") test.Assert(t, strings.Contains(err.Error(), "repository is already locked exclusively"), "create normal lock with exclusively locked repo didn't return the correct error") - test.Assert(t, retryLock <= duration && duration < retryLock+5*time.Millisecond, + test.Assert(t, retryLock <= duration && duration < retryLock+50*time.Millisecond, "create normal lock with exclusively locked repo didn't wait for the specified timeout") test.OK(t, lock.Unlock()) @@ -215,7 +215,7 @@ func TestLockWaitCancel(t *testing.T) { "create normal lock with exclusively locked repo didn't return an error") test.Assert(t, strings.Contains(err.Error(), "context canceled"), "create normal lock with exclusively locked repo didn't return the correct error") - test.Assert(t, cancelAfter <= duration && duration < cancelAfter+5*time.Millisecond, + test.Assert(t, cancelAfter <= duration && duration < cancelAfter+50*time.Millisecond, "create normal lock with exclusively locked repo didn't return in time") test.OK(t, lock.Unlock()) From 9f9e91eb0d402f55234e8fa564e7903ad679c727 Mon Sep 17 00:00:00 2001 From: Torben Giesselmann Date: Sun, 9 Apr 2023 11:47:10 -0700 Subject: [PATCH 033/127] Fix comment to comply with linter --- internal/restic/testing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/restic/testing.go b/internal/restic/testing.go index 9c21b81d1..dda9eff44 100644 --- a/internal/restic/testing.go +++ b/internal/restic/testing.go @@ -213,7 +213,7 @@ func TestSetSnapshotID(t testing.TB, sn *Snapshot, id ID) { sn.id = &id } -// Convenience function that parses a duration from a string or panics if string is invalid. +// ParseDurationOrPanic parses a duration from a string or panics if string is invalid. // The format is `6y5m234d37h`. func ParseDurationOrPanic(s string) Duration { d, err := ParseDuration(s) From 309cf0586ad6f36a7bd99e331cacfe9fbbea2a65 Mon Sep 17 00:00:00 2001 From: Torben Giesselmann Date: Sun, 9 Apr 2023 12:05:15 -0700 Subject: [PATCH 034/127] Add changelog entry --- changelog/unreleased/issue-2565 | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 changelog/unreleased/issue-2565 diff --git a/changelog/unreleased/issue-2565 b/changelog/unreleased/issue-2565 new file mode 100644 index 000000000..4150dcda4 --- /dev/null +++ b/changelog/unreleased/issue-2565 @@ -0,0 +1,8 @@ +Bugfix: Restic forget --keep-* options now interpret "-1" as "forever" + +Restic would forget snapshots that should have been kept when "-1" was +used as a value for --keep-* options. It now interprets "-1" as forever, +e.g. an option like --keep-monthly -1 will keep all monthly snapshots. + +https://github.com/restic/restic/issues/2565 +https://github.com/restic/restic/pull/4234 From 403b7ca2be321d7bb0092e6b42e5709b95b4eb03 Mon Sep 17 00:00:00 2001 From: Torben Giesselmann Date: Sun, 9 Apr 2023 12:57:37 -0700 Subject: [PATCH 035/127] forget: Update documentation --- doc/060_forget.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/060_forget.rst b/doc/060_forget.rst index b960ddb14..2353ef6a0 100644 --- a/doc/060_forget.rst +++ b/doc/060_forget.rst @@ -205,6 +205,7 @@ The ``forget`` command accepts the following policy options: natural time boundaries and *not* relative to when you run ``forget``. Weeks are Monday 00:00 to Sunday 23:59, days 00:00 to 23:59, hours :00 to :59, etc. They also only count hours/days/weeks/etc which have one or more snapshots. + A value of ``-1`` will be interpreted as "forever", i.e. "keep all". .. note:: All duration related options (``--keep-{within,-*}``) ignore snapshots with a timestamp in the future (relative to when the ``forget`` command is From ce51d2f3c07b2ebb831e29d8a3a2c7764e1ae93f Mon Sep 17 00:00:00 2001 From: Torben Giesselmann Date: Sun, 9 Apr 2023 12:59:15 -0700 Subject: [PATCH 036/127] forget: Update usage text --- cmd/restic/cmd_forget.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index ac3601584..cc1b4f6d8 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -67,12 +67,12 @@ func init() { cmdRoot.AddCommand(cmdForget) f := cmdForget.Flags() - f.IntVarP(&forgetOptions.Last, "keep-last", "l", 0, "keep the last `n` snapshots") - f.IntVarP(&forgetOptions.Hourly, "keep-hourly", "H", 0, "keep the last `n` hourly snapshots") - f.IntVarP(&forgetOptions.Daily, "keep-daily", "d", 0, "keep the last `n` daily snapshots") - f.IntVarP(&forgetOptions.Weekly, "keep-weekly", "w", 0, "keep the last `n` weekly snapshots") - f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots") - f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots") + f.IntVarP(&forgetOptions.Last, "keep-last", "l", 0, "keep the last `n` snapshots (use \"-1\" for n to keep all snapshots)") + f.IntVarP(&forgetOptions.Hourly, "keep-hourly", "H", 0, "keep the last `n` hourly snapshots (use \"-1\" for n to keep all hourly snapshots)") + f.IntVarP(&forgetOptions.Daily, "keep-daily", "d", 0, "keep the last `n` daily snapshots (use \"-1\" for n to keep all daily snapshots)") + f.IntVarP(&forgetOptions.Weekly, "keep-weekly", "w", 0, "keep the last `n` weekly snapshots (use \"-1\" for n to keep all weekly snapshots)") + f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots (use \"-1\" for n to keep all monthly snapshots)") + f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots (use \"-1\" for n to keep all yearly snapshots)") f.VarP(&forgetOptions.Within, "keep-within", "", "keep snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot") f.VarP(&forgetOptions.WithinHourly, "keep-within-hourly", "", "keep hourly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot") f.VarP(&forgetOptions.WithinDaily, "keep-within-daily", "", "keep daily snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot") From 71537da4b0934bda70c837758c187be2b4922c54 Mon Sep 17 00:00:00 2001 From: Torben Giesselmann Date: Sun, 9 Apr 2023 14:59:07 -0700 Subject: [PATCH 037/127] init: Add --copy-chunker-params verbose msg --- cmd/restic/cmd_init.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/restic/cmd_init.go b/cmd/restic/cmd_init.go index 2932870e8..a878f3e16 100644 --- a/cmd/restic/cmd_init.go +++ b/cmd/restic/cmd_init.go @@ -102,7 +102,12 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args [] } if !gopts.JSON { - Verbosef("created restic repository %v at %s\n", s.Config().ID[:10], location.StripPassword(gopts.Repo)) + Verbosef("created restic repository %v at %s", s.Config().ID[:10], location.StripPassword(gopts.Repo)) + if opts.CopyChunkerParameters && chunkerPolynomial != nil { + Verbosef(" with chunker parameters copied from secondary repository\n") + } else { + Verbosef("\n") + } Verbosef("\n") Verbosef("Please note that knowledge of your password is required to access\n") Verbosef("the repository. Losing your password means that your data is\n") From ba33e410685c3ff93f08124b8a31aaf1f9a495bb Mon Sep 17 00:00:00 2001 From: Torsten C <5834405+TorstenC@users.noreply.github.com> Date: Thu, 6 Apr 2023 18:32:09 +0200 Subject: [PATCH 038/127] 040_backup.rst line 228 - #4280 --exclude-caches Proposal for issue #4280 --- doc/040_backup.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 6dcb85b8a..bbb9df892 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -225,7 +225,7 @@ the exclude options are: - ``--exclude`` Specified one or more times to exclude one or more items - ``--iexclude`` Same as ``--exclude`` but ignores the case of paths -- ``--exclude-caches`` Specified once to exclude folders containing `this special file `__ +- ``--exclude-caches`` Specified once to exclude a folder's content if it contains `the special CACHEDIR.TAG file `__, but keep ``CACHEDIR.TAG``. - ``--exclude-file`` Specified one or more times to exclude items listed in a given file - ``--iexclude-file`` Same as ``exclude-file`` but ignores cases like in ``--iexclude`` - ``--exclude-if-present foo`` Specified one or more times to exclude a folder's content if it contains a file called ``foo`` (optionally having a given header, no wildcards for the file name supported) From 37aca6bec0837e89e87c18c7926b5bce48439fc0 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 12 Apr 2023 21:37:37 +0200 Subject: [PATCH 039/127] Add warnings via Warnf to the debug log --- cmd/restic/global.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 32f18a67f..a45ced8e8 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -282,6 +282,7 @@ func Warnf(format string, args ...interface{}) { if err != nil { fmt.Fprintf(os.Stderr, "unable to write to stderr: %v\n", err) } + debug.Log(format, args...) } // resolvePassword determines the password to be used for opening the repository. From e604939e72e84e470b146015614468790b6df672 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 12 Apr 2023 21:58:29 +0200 Subject: [PATCH 040/127] Debug log status code if execution is interrupted Currently, there is no clear indication in the debug log whether restic exited or not. --- cmd/restic/cleanup.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/restic/cleanup.go b/cmd/restic/cleanup.go index 61af72802..967957106 100644 --- a/cmd/restic/cleanup.go +++ b/cmd/restic/cleanup.go @@ -78,5 +78,6 @@ func CleanupHandler(c <-chan os.Signal) { // given exit code. func Exit(code int) { code = RunCleanupHandlers(code) + debug.Log("exiting with status code %d", code) os.Exit(code) } From 087cf7e114d9daf63e3a09e2a3e9d4023894fb17 Mon Sep 17 00:00:00 2001 From: "Heiko Schlittermann (HS12-RIPE)" Date: Wed, 22 Mar 2023 08:18:08 +0100 Subject: [PATCH 041/127] generate: write progress to STDOUT if this is a terminal This allows usage as: eval "$(restic generated --bash-completion /dev/stdout)" --- cmd/restic/cmd_generate.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/cmd/restic/cmd_generate.go b/cmd/restic/cmd_generate.go index 959a9d518..2e944ad37 100644 --- a/cmd/restic/cmd_generate.go +++ b/cmd/restic/cmd_generate.go @@ -63,22 +63,30 @@ func writeManpages(dir string) error { } func writeBashCompletion(file string) error { - Verbosef("writing bash completion file to %v\n", file) + if stdoutIsTerminal() { + Verbosef("writing bash completion file to %v\n", file) + } return cmdRoot.GenBashCompletionFile(file) } func writeFishCompletion(file string) error { - Verbosef("writing fish completion file to %v\n", file) + if stdoutIsTerminal() { + Verbosef("writing fish completion file to %v\n", file) + } return cmdRoot.GenFishCompletionFile(file, true) } func writeZSHCompletion(file string) error { - Verbosef("writing zsh completion file to %v\n", file) + if stdoutIsTerminal() { + Verbosef("writing zsh completion file to %v\n", file) + } return cmdRoot.GenZshCompletionFile(file) } func writePowerShellCompletion(file string) error { - Verbosef("writing powershell completion file to %v\n", file) + if stdoutIsTerminal() { + Verbosef("writing powershell completion file to %v\n", file) + } return cmdRoot.GenPowerShellCompletionFile(file) } From 21ad357c107a98baec3541b5518da9023e9ea095 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 28 Jan 2023 20:19:07 +0100 Subject: [PATCH 042/127] add linux/riscv64 builds --- .github/workflows/tests.yml | 2 +- changelog/unreleased/pull-4180 | 5 +++++ helpers/build-release-binaries/main.go | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/pull-4180 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4ef827843..9c9555543 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -197,7 +197,7 @@ jobs: matrix: # run cross-compile in three batches parallel so the overall tests run faster targets: - - "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le linux/s390x" + - "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le linux/riscv64 linux/s390x" - "openbsd/386 openbsd/amd64 \ freebsd/386 freebsd/amd64 freebsd/arm \ diff --git a/changelog/unreleased/pull-4180 b/changelog/unreleased/pull-4180 new file mode 100644 index 000000000..ff43feb2b --- /dev/null +++ b/changelog/unreleased/pull-4180 @@ -0,0 +1,5 @@ +Enhancement: Add release binaries for riscv64 architecture on Linux + +We've added release binaries for riscv64 architecture on Linux. + +https://github.com/restic/restic/pull/4180 diff --git a/helpers/build-release-binaries/main.go b/helpers/build-release-binaries/main.go index 1662ada0b..0c0015f42 100644 --- a/helpers/build-release-binaries/main.go +++ b/helpers/build-release-binaries/main.go @@ -225,7 +225,7 @@ var defaultBuildTargets = map[string][]string{ "aix": {"ppc64"}, "darwin": {"amd64", "arm64"}, "freebsd": {"386", "amd64", "arm"}, - "linux": {"386", "amd64", "arm", "arm64", "ppc64le", "mips", "mipsle", "mips64", "mips64le", "s390x"}, + "linux": {"386", "amd64", "arm", "arm64", "ppc64le", "mips", "mipsle", "mips64", "mips64le", "riscv64", "s390x"}, "netbsd": {"386", "amd64"}, "openbsd": {"386", "amd64"}, "windows": {"386", "amd64"}, From 74f7dd0b38a8505943b5dd2ad938932c4f8d35a5 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 6 Feb 2023 22:11:21 +0100 Subject: [PATCH 043/127] Make help for --verbose less confusing The output is now ``` -v, --verbose be verbose (specify multiple times or a level using --verbose=n, max level/times is 2) ``` instead of ``` -v, --verbose n be verbose (specify multiple times or a level using --verbose=n, max level/times is 2) ``` --- cmd/restic/global.go | 3 ++- doc/manual_rest.rst | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 517388e8d..b32265275 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -112,7 +112,8 @@ func init() { f.StringVarP(&globalOptions.KeyHint, "key-hint", "", "", "`key` ID of key to try decrypting first (default: $RESTIC_KEY_HINT)") f.StringVarP(&globalOptions.PasswordCommand, "password-command", "", "", "shell `command` to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)") f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not output comprehensive progress report") - f.CountVarP(&globalOptions.Verbose, "verbose", "v", "be verbose (specify multiple times or a level using --verbose=`n`, max level/times is 2)") + // use empty paremeter name as `-v, --verbose n` instead of the correct `--verbose=n` is confusing + f.CountVarP(&globalOptions.Verbose, "verbose", "v", "be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2)") f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repository, this allows some operations on read-only repositories") f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it") f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache `directory`. (default: use system default cache directory)") diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index 899a45688..97480db80 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -67,7 +67,7 @@ Usage help is available: -r, --repo repository repository to backup to or restore from (default: $RESTIC_REPOSITORY) --repository-file file file to read the repository location from (default: $RESTIC_REPOSITORY_FILE) --tls-client-cert file path to a file containing PEM encoded TLS client certificate and private key - -v, --verbose n be verbose (specify multiple times or a level using --verbose=n, max level/times is 2) + -v, --verbose be verbose (specify multiple times or a level using --verbose=n, max level/times is 2) Use "restic [command] --help" for more information about a command. @@ -142,7 +142,7 @@ command: -r, --repo repository repository to backup to or restore from (default: $RESTIC_REPOSITORY) --repository-file file file to read the repository location from (default: $RESTIC_REPOSITORY_FILE) --tls-client-cert file path to a file containing PEM encoded TLS client certificate and private key - -v, --verbose n be verbose (specify multiple times or a level using --verbose=n, max level/times is 2) + -v, --verbose be verbose (specify multiple times or a level using --verbose=n, max level/times is 2) Subcommands that support showing progress information such as ``backup``, ``check`` and ``prune`` will do so unless the quiet flag ``-q`` or From 97274ecabd26157881c4ff8f3bf062c3e7d51c14 Mon Sep 17 00:00:00 2001 From: greatroar <61184462+greatroar@users.noreply.github.com> Date: Fri, 17 Feb 2023 16:13:46 +0100 Subject: [PATCH 044/127] cmd, restic: Refactor and fix snapshot filtering This turns snapshotFilterOptions from cmd into a restic.SnapshotFilter type and makes restic.FindFilteredSnapshot and FindFilteredSnapshots methods on that type. This fixes #4211 by ensuring that hosts and paths are named struct fields instead of unnamed function arguments in long lists of such. Timestamp limits are also included in the new type. To avoid too much pointer handling, the convention is that time zero means no limit. That's January 1st, year 1, 00:00 UTC, which is so unlikely a date that we can sacrifice it for simpler code. --- changelog/unreleased/issue-4211 | 8 ++++ cmd/restic/cmd_backup.go | 8 +++- cmd/restic/cmd_copy.go | 9 ++-- cmd/restic/cmd_dump.go | 10 ++-- cmd/restic/cmd_find.go | 6 +-- cmd/restic/cmd_forget.go | 6 +-- cmd/restic/cmd_ls.go | 10 ++-- cmd/restic/cmd_mount.go | 8 ++-- cmd/restic/cmd_restore.go | 10 ++-- cmd/restic/cmd_rewrite.go | 6 +-- cmd/restic/cmd_snapshots.go | 6 +-- cmd/restic/cmd_stats.go | 6 +-- cmd/restic/cmd_tag.go | 6 +-- cmd/restic/find.go | 30 +++++------- cmd/restic/integration_test.go | 4 +- internal/fuse/root.go | 4 +- internal/fuse/snapshots_dirstruct.go | 2 +- internal/restic/snapshot_find.go | 68 +++++++++++++++------------ internal/restic/snapshot_find_test.go | 18 +++---- 19 files changed, 126 insertions(+), 99 deletions(-) create mode 100644 changelog/unreleased/issue-4211 diff --git a/changelog/unreleased/issue-4211 b/changelog/unreleased/issue-4211 new file mode 100644 index 000000000..45b7aee83 --- /dev/null +++ b/changelog/unreleased/issue-4211 @@ -0,0 +1,8 @@ +Bugfix: Restic dump now interprets --host and --path correctly + +Restic dump previously confused its --host= and --path= +options: it looked for snapshots with paths called from hosts +called . It now treats the options as intended. + +https://github.com/restic/restic/issues/4211 +https://github.com/restic/restic/pull/4212 diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index e59f503db..ec901828b 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -439,7 +439,13 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup if snName == "" { snName = "latest" } - sn, err := restic.FindFilteredSnapshot(ctx, repo.Backend(), repo, []string{opts.Host}, []restic.TagList{}, targets, &timeStampLimit, snName) + f := restic.SnapshotFilter{ + Hosts: []string{opts.Host}, + Paths: targets, + TimestampLimit: timeStampLimit, + } + + sn, err := f.FindLatest(ctx, repo.Backend(), repo, snName) // Snapshot not found is ok if no explicit parent was set if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) { err = nil diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 14ab1917a..2f095972a 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -39,7 +39,7 @@ new destination repository using the "init" command. // CopyOptions bundles all options for the copy command. type CopyOptions struct { secondaryRepoOptions - snapshotFilterOptions + restic.SnapshotFilter } var copyOptions CopyOptions @@ -49,7 +49,7 @@ func init() { f := cmdCopy.Flags() initSecondaryRepoOptions(f, ©Options.secondaryRepoOptions, "destination", "to copy snapshots from") - initMultiSnapshotFilterOptions(f, ©Options.snapshotFilterOptions, true) + initMultiSnapshotFilter(f, ©Options.SnapshotFilter, true) } func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error { @@ -108,7 +108,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] } dstSnapshotByOriginal := make(map[restic.ID][]*restic.Snapshot) - for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, opts.Hosts, opts.Tags, opts.Paths, nil) { + for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, &opts.SnapshotFilter, nil) { if sn.Original != nil && !sn.Original.IsNull() { dstSnapshotByOriginal[*sn.Original] = append(dstSnapshotByOriginal[*sn.Original], sn) } @@ -119,8 +119,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] // remember already processed trees across all snapshots visitedTrees := restic.NewIDSet() - for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) { - + for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, &opts.SnapshotFilter, args) { // check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields srcOriginal := *sn.ID() if sn.Original != nil { diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index a480b12f4..cda7b65b9 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -40,7 +40,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er // DumpOptions collects all options for the dump command. type DumpOptions struct { - snapshotFilterOptions + restic.SnapshotFilter Archive string } @@ -50,7 +50,7 @@ func init() { cmdRoot.AddCommand(cmdDump) flags := cmdDump.Flags() - initSingleSnapshotFilterOptions(flags, &dumpOptions.snapshotFilterOptions) + initSingleSnapshotFilter(flags, &dumpOptions.SnapshotFilter) flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"") } @@ -139,7 +139,11 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] } } - sn, err := restic.FindFilteredSnapshot(ctx, repo.Backend(), repo, opts.Paths, opts.Tags, opts.Hosts, nil, snapshotIDString) + sn, err := (&restic.SnapshotFilter{ + Hosts: opts.Hosts, + Paths: opts.Paths, + Tags: opts.Tags, + }).FindLatest(ctx, repo.Backend(), repo, snapshotIDString) if err != nil { return errors.Fatalf("failed to find snapshot: %v", err) } diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 8e5f9b604..e5457c3be 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -51,7 +51,7 @@ type FindOptions struct { PackID, ShowPackID bool CaseInsensitive bool ListLong bool - snapshotFilterOptions + restic.SnapshotFilter } var findOptions FindOptions @@ -70,7 +70,7 @@ func init() { f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern") f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode") - initMultiSnapshotFilterOptions(f, &findOptions.snapshotFilterOptions, true) + initMultiSnapshotFilter(f, &findOptions.SnapshotFilter, true) } type findPattern struct { @@ -618,7 +618,7 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args [] } } - for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, opts.Snapshots) { + for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, opts.Snapshots) { if f.blobIDs != nil || f.treeIDs != nil { if err = f.findIDs(ctx, sn); err != nil && err.Error() != "OK" { return err diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index 472b22b79..e4e44a368 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -52,7 +52,7 @@ type ForgetOptions struct { WithinYearly restic.Duration KeepTags restic.TagLists - snapshotFilterOptions + restic.SnapshotFilter Compact bool // Grouping @@ -81,7 +81,7 @@ func init() { f.VarP(&forgetOptions.WithinYearly, "keep-within-yearly", "", "keep yearly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot") f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)") - initMultiSnapshotFilterOptions(f, &forgetOptions.snapshotFilterOptions, false) + initMultiSnapshotFilter(f, &forgetOptions.SnapshotFilter, false) f.StringArrayVar(&forgetOptions.Hosts, "hostname", nil, "only consider snapshots with the given `hostname` (can be specified multiple times)") err := f.MarkDeprecated("hostname", "use --host") if err != nil { @@ -126,7 +126,7 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg var snapshots restic.Snapshots removeSnIDs := restic.NewIDSet() - for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { + for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) { snapshots = append(snapshots, sn) } diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 7dd41ab21..aeaa750eb 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -49,7 +49,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er // LsOptions collects all options for the ls command. type LsOptions struct { ListLong bool - snapshotFilterOptions + restic.SnapshotFilter Recursive bool } @@ -59,7 +59,7 @@ func init() { cmdRoot.AddCommand(cmdLs) flags := cmdLs.Flags() - initSingleSnapshotFilterOptions(flags, &lsOptions.snapshotFilterOptions) + initSingleSnapshotFilter(flags, &lsOptions.SnapshotFilter) flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode") flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories") } @@ -210,7 +210,11 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri } } - sn, err := restic.FindFilteredSnapshot(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, nil, args[0]) + sn, err := (&restic.SnapshotFilter{ + Hosts: opts.Hosts, + Paths: opts.Paths, + Tags: opts.Tags, + }).FindLatest(ctx, snapshotLister, repo, args[0]) if err != nil { return err } diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index 7afb30f7c..0501bfe89 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -77,7 +77,7 @@ type MountOptions struct { OwnerRoot bool AllowOther bool NoDefaultPermissions bool - snapshotFilterOptions + restic.SnapshotFilter TimeTemplate string PathTemplates []string } @@ -92,7 +92,7 @@ func init() { mountFlags.BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory") mountFlags.BoolVar(&mountOptions.NoDefaultPermissions, "no-default-permissions", false, "for 'allow-other', ignore Unix permissions and allow users to read all snapshot files") - initMultiSnapshotFilterOptions(mountFlags, &mountOptions.snapshotFilterOptions, true) + initMultiSnapshotFilter(mountFlags, &mountOptions.SnapshotFilter, true) mountFlags.StringArrayVar(&mountOptions.PathTemplates, "path-template", nil, "set `template` for path names (can be specified multiple times)") mountFlags.StringVar(&mountOptions.TimeTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs") @@ -180,9 +180,7 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args cfg := fuse.Config{ OwnerIsRoot: opts.OwnerRoot, - Hosts: opts.Hosts, - Tags: opts.Tags, - Paths: opts.Paths, + Filter: opts.SnapshotFilter, TimeTemplate: opts.TimeTemplate, PathTemplates: opts.PathTemplates, } diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index b70cb52ff..579711662 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -42,7 +42,7 @@ type RestoreOptions struct { Include []string InsensitiveInclude []string Target string - snapshotFilterOptions + restic.SnapshotFilter Sparse bool Verify bool } @@ -59,7 +59,7 @@ func init() { flags.StringArrayVar(&restoreOptions.InsensitiveInclude, "iinclude", nil, "same as `--include` but ignores the casing of filenames") flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to") - initSingleSnapshotFilterOptions(flags, &restoreOptions.snapshotFilterOptions) + initSingleSnapshotFilter(flags, &restoreOptions.SnapshotFilter) flags.BoolVar(&restoreOptions.Sparse, "sparse", false, "restore files as sparse") flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content") } @@ -131,7 +131,11 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, a } } - sn, err := restic.FindFilteredSnapshot(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, nil, snapshotIDString) + sn, err := (&restic.SnapshotFilter{ + Hosts: opts.Hosts, + Paths: opts.Paths, + Tags: opts.Tags, + }).FindLatest(ctx, repo.Backend(), repo, snapshotIDString) if err != nil { return errors.Fatalf("failed to find snapshot: %v", err) } diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index cfe56db87..0d9aa1c8c 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -51,7 +51,7 @@ type RewriteOptions struct { Forget bool DryRun bool - snapshotFilterOptions + restic.SnapshotFilter excludePatternOptions } @@ -64,7 +64,7 @@ func init() { f.BoolVarP(&rewriteOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones") f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done") - initMultiSnapshotFilterOptions(f, &rewriteOptions.snapshotFilterOptions, true) + initMultiSnapshotFilter(f, &rewriteOptions.SnapshotFilter, true) initExcludePatternOptions(f, &rewriteOptions.excludePatternOptions) } @@ -186,7 +186,7 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a } changedCount := 0 - for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, args) { + for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) { Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) changed, err := rewriteSnapshot(ctx, repo, sn, opts) if err != nil { diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index 0bfa4d110..c5faa044a 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -32,7 +32,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er // SnapshotOptions bundles all options for the snapshots command. type SnapshotOptions struct { - snapshotFilterOptions + restic.SnapshotFilter Compact bool Last bool // This option should be removed in favour of Latest. Latest int @@ -45,7 +45,7 @@ func init() { cmdRoot.AddCommand(cmdSnapshots) f := cmdSnapshots.Flags() - initMultiSnapshotFilterOptions(f, &snapshotOptions.snapshotFilterOptions, true) + initMultiSnapshotFilter(f, &snapshotOptions.SnapshotFilter, true) f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact output format") f.BoolVar(&snapshotOptions.Last, "last", false, "only show the last snapshot for each host and path") err := f.MarkDeprecated("last", "use --latest 1") @@ -73,7 +73,7 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions } var snapshots restic.Snapshots - for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { + for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) { snapshots = append(snapshots, sn) } snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy) diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 99d16b932..55ba6f254 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -58,7 +58,7 @@ type StatsOptions struct { // the mode of counting to perform (see consts for available modes) countMode string - snapshotFilterOptions + restic.SnapshotFilter } var statsOptions StatsOptions @@ -67,7 +67,7 @@ func init() { cmdRoot.AddCommand(cmdStats) f := cmdStats.Flags() f.StringVar(&statsOptions.countMode, "mode", countModeRestoreSize, "counting mode: restore-size (default), files-by-contents, blobs-per-file or raw-data") - initMultiSnapshotFilterOptions(f, &statsOptions.snapshotFilterOptions, true) + initMultiSnapshotFilter(f, &statsOptions.SnapshotFilter, true) } func runStats(ctx context.Context, gopts GlobalOptions, args []string) error { @@ -111,7 +111,7 @@ func runStats(ctx context.Context, gopts GlobalOptions, args []string) error { SnapshotsCount: 0, } - for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, statsOptions.Hosts, statsOptions.Tags, statsOptions.Paths, args) { + for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &statsOptions.SnapshotFilter, args) { err = statsWalkSnapshot(ctx, sn, repo, stats) if err != nil { return fmt.Errorf("error walking snapshot: %v", err) diff --git a/cmd/restic/cmd_tag.go b/cmd/restic/cmd_tag.go index 222ddd04a..e5948ea02 100644 --- a/cmd/restic/cmd_tag.go +++ b/cmd/restic/cmd_tag.go @@ -35,7 +35,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er // TagOptions bundles all options for the 'tag' command. type TagOptions struct { - snapshotFilterOptions + restic.SnapshotFilter SetTags restic.TagLists AddTags restic.TagLists RemoveTags restic.TagLists @@ -50,7 +50,7 @@ func init() { tagFlags.Var(&tagOptions.SetTags, "set", "`tags` which will replace the existing tags in the format `tag[,tag,...]` (can be given multiple times)") tagFlags.Var(&tagOptions.AddTags, "add", "`tags` which will be added to the existing tags in the format `tag[,tag,...]` (can be given multiple times)") tagFlags.Var(&tagOptions.RemoveTags, "remove", "`tags` which will be removed from the existing tags in the format `tag[,tag,...]` (can be given multiple times)") - initMultiSnapshotFilterOptions(tagFlags, &tagOptions.snapshotFilterOptions, true) + initMultiSnapshotFilter(tagFlags, &tagOptions.SnapshotFilter, true) } func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, setTags, addTags, removeTags []string) (bool, error) { @@ -119,7 +119,7 @@ func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []st } changeCnt := 0 - for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { + for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) { changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten()) if err != nil { Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err) diff --git a/cmd/restic/find.go b/cmd/restic/find.go index 7b488c7aa..54d3563b1 100644 --- a/cmd/restic/find.go +++ b/cmd/restic/find.go @@ -8,34 +8,28 @@ import ( "github.com/spf13/pflag" ) -type snapshotFilterOptions struct { - Hosts []string - Tags restic.TagLists - Paths []string -} - -// initMultiSnapshotFilterOptions is used for commands that work on multiple snapshots +// initMultiSnapshotFilter is used for commands that work on multiple snapshots // MUST be combined with restic.FindFilteredSnapshots or FindFilteredSnapshots -func initMultiSnapshotFilterOptions(flags *pflag.FlagSet, options *snapshotFilterOptions, addHostShorthand bool) { +func initMultiSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter, addHostShorthand bool) { hostShorthand := "H" if !addHostShorthand { hostShorthand = "" } - flags.StringArrayVarP(&options.Hosts, "host", hostShorthand, nil, "only consider snapshots for this `host` (can be specified multiple times)") - flags.Var(&options.Tags, "tag", "only consider snapshots including `tag[,tag,...]` (can be specified multiple times)") - flags.StringArrayVar(&options.Paths, "path", nil, "only consider snapshots including this (absolute) `path` (can be specified multiple times)") + flags.StringArrayVarP(&filt.Hosts, "host", hostShorthand, nil, "only consider snapshots for this `host` (can be specified multiple times)") + flags.Var(&filt.Tags, "tag", "only consider snapshots including `tag[,tag,...]` (can be specified multiple times)") + flags.StringArrayVar(&filt.Paths, "path", nil, "only consider snapshots including this (absolute) `path` (can be specified multiple times)") } -// initSingleSnapshotFilterOptions is used for commands that work on a single snapshot +// initSingleSnapshotFilter is used for commands that work on a single snapshot // MUST be combined with restic.FindFilteredSnapshot -func initSingleSnapshotFilterOptions(flags *pflag.FlagSet, options *snapshotFilterOptions) { - flags.StringArrayVarP(&options.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times)") - flags.Var(&options.Tags, "tag", "only consider snapshots including `tag[,tag,...]`, when snapshot ID \"latest\" is given (can be specified multiple times)") - flags.StringArrayVar(&options.Paths, "path", nil, "only consider snapshots including this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times)") +func initSingleSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter) { + flags.StringArrayVarP(&filt.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times)") + flags.Var(&filt.Tags, "tag", "only consider snapshots including `tag[,tag,...]`, when snapshot ID \"latest\" is given (can be specified multiple times)") + flags.StringArrayVar(&filt.Paths, "path", nil, "only consider snapshots including this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times)") } // FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. -func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, hosts []string, tags []restic.TagList, paths []string, snapshotIDs []string) <-chan *restic.Snapshot { +func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, f *restic.SnapshotFilter, snapshotIDs []string) <-chan *restic.Snapshot { out := make(chan *restic.Snapshot) go func() { defer close(out) @@ -45,7 +39,7 @@ func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic. return } - err = restic.FindFilteredSnapshots(ctx, be, loader, hosts, tags, paths, snapshotIDs, func(id string, sn *restic.Snapshot, err error) error { + err = f.FindAll(ctx, be, loader, snapshotIDs, func(id string, sn *restic.Snapshot, err error) error { if err != nil { Warnf("Ignoring %q: %v\n", id, err) } else { diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 062a5954c..c87722f02 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -106,7 +106,7 @@ func testRunRestore(t testing.TB, opts GlobalOptions, dir string, snapshotID res func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths []string, hosts []string) { opts := RestoreOptions{ Target: dir, - snapshotFilterOptions: snapshotFilterOptions{ + SnapshotFilter: restic.SnapshotFilter{ Hosts: hosts, Paths: paths, }, @@ -2196,7 +2196,7 @@ func TestFindListOnce(t *testing.T) { snapshotIDs := restic.NewIDSet() // specify the two oldest snapshots explicitly and use "latest" to reference the newest one - for sn := range FindFilteredSnapshots(context.TODO(), repo.Backend(), repo, nil, nil, nil, []string{ + for sn := range FindFilteredSnapshots(context.TODO(), repo.Backend(), repo, &restic.SnapshotFilter{}, []string{ secondSnapshot[0].String(), secondSnapshot[1].String()[:8], "latest", diff --git a/internal/fuse/root.go b/internal/fuse/root.go index fc8841964..ab6116f0d 100644 --- a/internal/fuse/root.go +++ b/internal/fuse/root.go @@ -16,9 +16,7 @@ import ( // Config holds settings for the fuse mount. type Config struct { OwnerIsRoot bool - Hosts []string - Tags []restic.TagList - Paths []string + Filter restic.SnapshotFilter TimeTemplate string PathTemplates []string } diff --git a/internal/fuse/snapshots_dirstruct.go b/internal/fuse/snapshots_dirstruct.go index f8e66d076..3080d4de8 100644 --- a/internal/fuse/snapshots_dirstruct.go +++ b/internal/fuse/snapshots_dirstruct.go @@ -295,7 +295,7 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error { } var snapshots restic.Snapshots - err := restic.FindFilteredSnapshots(ctx, d.root.repo.Backend(), d.root.repo, d.root.cfg.Hosts, d.root.cfg.Tags, d.root.cfg.Paths, nil, func(id string, sn *restic.Snapshot, err error) error { + err := d.root.cfg.Filter.FindAll(ctx, d.root.repo.Backend(), d.root.repo, nil, func(id string, sn *restic.Snapshot, err error) error { if sn != nil { snapshots = append(snapshots, sn) } diff --git a/internal/restic/snapshot_find.go b/internal/restic/snapshot_find.go index 4f8231a7f..4d4bb4957 100644 --- a/internal/restic/snapshot_find.go +++ b/internal/restic/snapshot_find.go @@ -12,13 +12,32 @@ import ( // ErrNoSnapshotFound is returned when no snapshot for the given criteria could be found. var ErrNoSnapshotFound = errors.New("no snapshot found") -// findLatestSnapshot finds latest snapshot with optional target/directory, tags, hostname, and timestamp filters. -func findLatestSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, hosts []string, - tags []TagList, paths []string, timeStampLimit *time.Time) (*Snapshot, error) { +// A SnapshotFilter denotes a set of snapshots based on hosts, tags and paths. +type SnapshotFilter struct { + _ struct{} // Force naming fields in literals. + + Hosts []string + Tags TagLists + Paths []string + // Match snapshots from before this timestamp. Zero for no limit. + TimestampLimit time.Time +} + +func (f *SnapshotFilter) empty() bool { + return len(f.Hosts)+len(f.Tags)+len(f.Paths) == 0 +} + +func (f *SnapshotFilter) matches(sn *Snapshot) bool { + return sn.HasHostname(f.Hosts) && sn.HasTagList(f.Tags) && sn.HasPaths(f.Paths) +} + +// findLatest finds the latest snapshot with optional target/directory, +// tags, hostname, and timestamp filters. +func (f *SnapshotFilter) findLatest(ctx context.Context, be Lister, loader LoaderUnpacked) (*Snapshot, error) { var err error - absTargets := make([]string, 0, len(paths)) - for _, target := range paths { + absTargets := make([]string, 0, len(f.Paths)) + for _, target := range f.Paths { if !filepath.IsAbs(target) { target, err = filepath.Abs(target) if err != nil { @@ -35,7 +54,7 @@ func findLatestSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, h return errors.Errorf("Error loading snapshot %v: %v", id.Str(), err) } - if timeStampLimit != nil && snapshot.Time.After(*timeStampLimit) { + if !f.TimestampLimit.IsZero() && snapshot.Time.After(f.TimestampLimit) { return nil } @@ -43,15 +62,7 @@ func findLatestSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, h return nil } - if !snapshot.HasHostname(hosts) { - return nil - } - - if !snapshot.HasTagList(tags) { - return nil - } - - if !snapshot.HasPaths(absTargets) { + if !f.matches(snapshot) { return nil } @@ -85,12 +96,14 @@ func FindSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, s strin return LoadSnapshot(ctx, loader, id) } -// FindFilteredSnapshot returns either the latests from a filtered list of all snapshots or a snapshot specified by `snapshotID`. -func FindFilteredSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, hosts []string, tags []TagList, paths []string, timeStampLimit *time.Time, snapshotID string) (*Snapshot, error) { +// FindLatest returns either the latest of a filtered list of all snapshots +// or a snapshot specified by `snapshotID`. +func (f *SnapshotFilter) FindLatest(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotID string) (*Snapshot, error) { if snapshotID == "latest" { - sn, err := findLatestSnapshot(ctx, be, loader, hosts, tags, paths, timeStampLimit) + sn, err := f.findLatest(ctx, be, loader) if err == ErrNoSnapshotFound { - err = fmt.Errorf("snapshot filter (Paths:%v Tags:%v Hosts:%v): %w", paths, tags, hosts, err) + err = fmt.Errorf("snapshot filter (Paths:%v Tags:%v Hosts:%v): %w", + f.Paths, f.Tags, f.Hosts, err) } return sn, err } @@ -99,8 +112,8 @@ func FindFilteredSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, type SnapshotFindCb func(string, *Snapshot, error) error -// FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. -func FindFilteredSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked, hosts []string, tags []TagList, paths []string, snapshotIDs []string, fn SnapshotFindCb) error { +// FindAll yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. +func (f *SnapshotFilter) FindAll(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotIDs []string, fn SnapshotFindCb) error { if len(snapshotIDs) != 0 { var err error usedFilter := false @@ -116,9 +129,10 @@ func FindFilteredSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked usedFilter = true - sn, err = findLatestSnapshot(ctx, be, loader, hosts, tags, paths, nil) + sn, err = f.findLatest(ctx, be, loader) if err == ErrNoSnapshotFound { - err = errors.Errorf("no snapshot matched given filter (Paths:%v Tags:%v Hosts:%v)", paths, tags, hosts) + err = errors.Errorf("no snapshot matched given filter (Paths:%v Tags:%v Hosts:%v)", + f.Paths, f.Tags, f.Hosts) } if sn != nil { ids.Insert(*sn.ID()) @@ -141,18 +155,14 @@ func FindFilteredSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked } // Give the user some indication their filters are not used. - if !usedFilter && (len(hosts) != 0 || len(tags) != 0 || len(paths) != 0) { + if !usedFilter && !f.empty() { return fn("filters", nil, errors.Errorf("explicit snapshot ids are given")) } return nil } return ForAllSnapshots(ctx, be, loader, nil, func(id ID, sn *Snapshot, err error) error { - if err != nil { - return fn(id.String(), sn, err) - } - - if !sn.HasHostname(hosts) || !sn.HasTagList(tags) || !sn.HasPaths(paths) { + if err == nil && !f.matches(sn) { return nil } diff --git a/internal/restic/snapshot_find_test.go b/internal/restic/snapshot_find_test.go index 3c587dde1..d098b5224 100644 --- a/internal/restic/snapshot_find_test.go +++ b/internal/restic/snapshot_find_test.go @@ -14,13 +14,14 @@ func TestFindLatestSnapshot(t *testing.T) { restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1, 0) latestSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1, 0) - sn, err := restic.FindFilteredSnapshot(context.TODO(), repo.Backend(), repo, []string{"foo"}, []restic.TagList{}, []string{}, nil, "latest") + f := restic.SnapshotFilter{Hosts: []string{"foo"}} + sn, err := f.FindLatest(context.TODO(), repo.Backend(), repo, "latest") if err != nil { - t.Fatalf("FindLatestSnapshot returned error: %v", err) + t.Fatalf("FindLatest returned error: %v", err) } if *sn.ID() != *latestSnapshot.ID() { - t.Errorf("FindLatestSnapshot returned wrong snapshot ID: %v", *sn.ID()) + t.Errorf("FindLatest returned wrong snapshot ID: %v", *sn.ID()) } } @@ -30,14 +31,15 @@ func TestFindLatestSnapshotWithMaxTimestamp(t *testing.T) { desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1, 0) restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1, 0) - maxTimestamp := parseTimeUTC("2018-08-08 08:08:08") - - sn, err := restic.FindFilteredSnapshot(context.TODO(), repo.Backend(), repo, []string{"foo"}, []restic.TagList{}, []string{}, &maxTimestamp, "latest") + sn, err := (&restic.SnapshotFilter{ + Hosts: []string{"foo"}, + TimestampLimit: parseTimeUTC("2018-08-08 08:08:08"), + }).FindLatest(context.TODO(), repo.Backend(), repo, "latest") if err != nil { - t.Fatalf("FindLatestSnapshot returned error: %v", err) + t.Fatalf("FindLatest returned error: %v", err) } if *sn.ID() != *desiredSnapshot.ID() { - t.Errorf("FindLatestSnapshot returned wrong snapshot ID: %v", *sn.ID()) + t.Errorf("FindLatest returned wrong snapshot ID: %v", *sn.ID()) } } From 593eb710b413fe190477d6fa296aef1fcb662525 Mon Sep 17 00:00:00 2001 From: Ian Muge Date: Tue, 21 Feb 2023 11:28:52 +0100 Subject: [PATCH 045/127] added changelog --- changelog/unreleased/pull-4219 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/pull-4219 diff --git a/changelog/unreleased/pull-4219 b/changelog/unreleased/pull-4219 new file mode 100644 index 000000000..7d20c3607 --- /dev/null +++ b/changelog/unreleased/pull-4219 @@ -0,0 +1,5 @@ +Enhancement: Upgrade Minio to 7.0.49 + +Upgraded to allow use of the ap-southeast-4 region (Melbourne) + +https://github.com/restic/restic/pull/4219 From 9412f37e50a0ea8ba12e845a2f84871255e66b9d Mon Sep 17 00:00:00 2001 From: greatroar <61184462+greatroar@users.noreply.github.com> Date: Sat, 11 Feb 2023 14:51:58 +0100 Subject: [PATCH 046/127] ui/termstatus: Quote funny filenames Fixes #2260, #4191. --- changelog/unreleased/issue-2260 | 13 +++++++++++ internal/ui/backup/text.go | 2 ++ internal/ui/termstatus/status.go | 29 ++++++++++++++++++------ internal/ui/termstatus/status_test.go | 32 ++++++++++++++++++++++++++- 4 files changed, 68 insertions(+), 8 deletions(-) create mode 100644 changelog/unreleased/issue-2260 diff --git a/changelog/unreleased/issue-2260 b/changelog/unreleased/issue-2260 new file mode 100644 index 000000000..96c79a035 --- /dev/null +++ b/changelog/unreleased/issue-2260 @@ -0,0 +1,13 @@ +Bugfix: Exotic filenames no longer break restic backup's status output + +Restic backup shows the names of files that it is working on. In previous +versions of restic, those names were printed without first sanitizing them, +so that filenames containing newlines or terminal control characters could +mess up restic backup's output or even change the state of a terminal. + +Filenames are now checked and quoted if they contain non-printable or +non-Unicode characters. + +https://github.com/restic/restic/issues/2260 +https://github.com/restic/restic/issues/4191 +https://github.com/restic/restic/pull/4192 diff --git a/internal/ui/backup/text.go b/internal/ui/backup/text.go index 0c5f897dd..acb2a8d3a 100644 --- a/internal/ui/backup/text.go +++ b/internal/ui/backup/text.go @@ -86,6 +86,8 @@ func (b *TextProgress) Error(item string, err error) error { // CompleteItem is the status callback function for the archiver when a // file/dir has been saved successfully. func (b *TextProgress) CompleteItem(messageType, item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration) { + item = termstatus.Quote(item) + switch messageType { case "dir new": b.VV("new %v, saved in %.3fs (%v added, %v stored, %v metadata)", diff --git a/internal/ui/termstatus/status.go b/internal/ui/termstatus/status.go index fdc7e14f6..a1b7a5fcc 100644 --- a/internal/ui/termstatus/status.go +++ b/internal/ui/termstatus/status.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "os" + "strconv" "strings" "unicode" @@ -325,6 +326,7 @@ func wideRune(r rune) bool { } // SetStatus updates the status lines. +// The lines should not contain newlines; this method adds them. func (t *Terminal) SetStatus(lines []string) { if len(lines) == 0 { return @@ -341,21 +343,34 @@ func (t *Terminal) SetStatus(lines []string) { } } - // make sure that all lines have a line break and are not too long + // Sanitize lines and truncate them if they're too long. for i, line := range lines { - line = strings.TrimRight(line, "\n") + line = Quote(line) if width > 0 { line = Truncate(line, width-2) } - lines[i] = line + "\n" + if i < len(lines)-1 { // Last line gets no line break. + lines[i] = line + "\n" + } } - // make sure the last line does not have a line break - last := len(lines) - 1 - lines[last] = strings.TrimRight(lines[last], "\n") - select { case t.status <- status{lines: lines}: case <-t.closed: } } + +// Quote lines with funny characters in them, meaning control chars, newlines, +// tabs, anything else non-printable and invalid UTF-8. +// +// This is intended to produce a string that does not mess up the terminal +// rather than produce an unambiguous quoted string. +func Quote(line string) string { + for _, r := range line { + // The replacement character usually means the input is not UTF-8. + if r == unicode.ReplacementChar || !unicode.IsPrint(r) { + return strconv.Quote(line) + } + } + return line +} diff --git a/internal/ui/termstatus/status_test.go b/internal/ui/termstatus/status_test.go index ce18f42e6..40a908deb 100644 --- a/internal/ui/termstatus/status_test.go +++ b/internal/ui/termstatus/status_test.go @@ -1,6 +1,36 @@ package termstatus -import "testing" +import ( + "strconv" + "testing" + + rtest "github.com/restic/restic/internal/test" +) + +func TestQuote(t *testing.T) { + for _, c := range []struct { + in string + needQuote bool + }{ + {"foo.bar/baz", false}, + {"föó_bàŕ-bãẑ", false}, + {" foo ", false}, + {"foo bar", false}, + {"foo\nbar", true}, + {"foo\rbar", true}, + {"foo\abar", true}, + {"\xff", true}, + {`c:\foo\bar`, false}, + // Issue #2260: terminal control characters. + {"\x1bm_red_is_beautiful", true}, + } { + if c.needQuote { + rtest.Equals(t, strconv.Quote(c.in), Quote(c.in)) + } else { + rtest.Equals(t, c.in, Quote(c.in)) + } + } +} func TestTruncate(t *testing.T) { var tests = []struct { From 49e32f3f8ae509ae818a6501cd9de51287eb586c Mon Sep 17 00:00:00 2001 From: greatroar <61184462+greatroar@users.noreply.github.com> Date: Fri, 14 Apr 2023 11:13:39 +0200 Subject: [PATCH 047/127] ui/termstatus: Optimize Truncate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit x/text/width.LookupRune has to re-encode its argument as UTF-8, while LookupString operates on the UTF-8 directly. The uint casts get rid of a bounds check. Benchmark results, with b.ResetTimer introduced first: name old time/op new time/op delta TruncateASCII-8 69.7ns ± 1% 55.2ns ± 1% -20.90% (p=0.000 n=20+18) TruncateUnicode-8 350ns ± 1% 171ns ± 1% -51.05% (p=0.000 n=20+19) --- internal/ui/termstatus/status.go | 27 ++++++++++++++++++--------- internal/ui/termstatus/status_test.go | 8 ++++++-- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/internal/ui/termstatus/status.go b/internal/ui/termstatus/status.go index fdc7e14f6..26fe8184b 100644 --- a/internal/ui/termstatus/status.go +++ b/internal/ui/termstatus/status.go @@ -302,26 +302,35 @@ func Truncate(s string, w int) string { return s } - for i, r := range s { + for i := uint(0); i < uint(len(s)); { + utfsize := uint(1) // UTF-8 encoding size of first rune in s. w-- - if r > unicode.MaxASCII && wideRune(r) { - w-- + + if s[i] > unicode.MaxASCII { + var wide bool + if wide, utfsize = wideRune(s[i:]); wide { + w-- + } } if w < 0 { return s[:i] } + i += utfsize } return s } -// Guess whether r would occupy two terminal cells instead of one. -// This cannot be determined exactly without knowing the terminal font, -// so we treat all ambigous runes as full-width, i.e., two cells. -func wideRune(r rune) bool { - kind := width.LookupRune(r).Kind() - return kind != width.Neutral && kind != width.EastAsianNarrow +// Guess whether the first rune in s would occupy two terminal cells +// instead of one. This cannot be determined exactly without knowing +// the terminal font, so we treat all ambigous runes as full-width, +// i.e., two cells. +func wideRune(s string) (wide bool, utfsize uint) { + prop, size := width.LookupString(s) + kind := prop.Kind() + wide = kind != width.Neutral && kind != width.EastAsianNarrow + return wide, uint(size) } // SetStatus updates the status lines. diff --git a/internal/ui/termstatus/status_test.go b/internal/ui/termstatus/status_test.go index ce18f42e6..696684502 100644 --- a/internal/ui/termstatus/status_test.go +++ b/internal/ui/termstatus/status_test.go @@ -49,11 +49,15 @@ func BenchmarkTruncateASCII(b *testing.B) { func BenchmarkTruncateUnicode(b *testing.B) { s := "Hello World or Καλημέρα κόσμε or こんにちは 世界" w := 0 - for _, r := range s { + for i := 0; i < len(s); { w++ - if wideRune(r) { + wide, utfsize := wideRune(s[i:]) + if wide { w++ } + i += int(utfsize) } + b.ResetTimer() + benchmarkTruncate(b, s, w-1) } From 6d6c04abef9929e1ce04f33a379386c4de3b819b Mon Sep 17 00:00:00 2001 From: Torben Giesselmann Date: Fri, 14 Apr 2023 10:05:23 -0700 Subject: [PATCH 048/127] forget: Simplify usage text --- cmd/restic/cmd_forget.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index cc1b4f6d8..3154a050e 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -67,12 +67,12 @@ func init() { cmdRoot.AddCommand(cmdForget) f := cmdForget.Flags() - f.IntVarP(&forgetOptions.Last, "keep-last", "l", 0, "keep the last `n` snapshots (use \"-1\" for n to keep all snapshots)") - f.IntVarP(&forgetOptions.Hourly, "keep-hourly", "H", 0, "keep the last `n` hourly snapshots (use \"-1\" for n to keep all hourly snapshots)") - f.IntVarP(&forgetOptions.Daily, "keep-daily", "d", 0, "keep the last `n` daily snapshots (use \"-1\" for n to keep all daily snapshots)") - f.IntVarP(&forgetOptions.Weekly, "keep-weekly", "w", 0, "keep the last `n` weekly snapshots (use \"-1\" for n to keep all weekly snapshots)") - f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots (use \"-1\" for n to keep all monthly snapshots)") - f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots (use \"-1\" for n to keep all yearly snapshots)") + f.IntVarP(&forgetOptions.Last, "keep-last", "l", 0, "keep the last `n` snapshots (use '-1' to keep all snapshots)") + f.IntVarP(&forgetOptions.Hourly, "keep-hourly", "H", 0, "keep the last `n` hourly snapshots (use '-1' to keep all hourly snapshots)") + f.IntVarP(&forgetOptions.Daily, "keep-daily", "d", 0, "keep the last `n` daily snapshots (use '-1' to keep all daily snapshots)") + f.IntVarP(&forgetOptions.Weekly, "keep-weekly", "w", 0, "keep the last `n` weekly snapshots (use '-1' to keep all weekly snapshots)") + f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots (use '-1' to keep all monthly snapshots)") + f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots (use '-1' to keep all yearly snapshots)") f.VarP(&forgetOptions.Within, "keep-within", "", "keep snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot") f.VarP(&forgetOptions.WithinHourly, "keep-within-hourly", "", "keep hourly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot") f.VarP(&forgetOptions.WithinDaily, "keep-within-daily", "", "keep daily snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot") From 4304e01ca21885f0fa5a31106075c6a11c27c5e3 Mon Sep 17 00:00:00 2001 From: greatroar <61184462+greatroar@users.noreply.github.com> Date: Tue, 7 Mar 2023 22:12:08 +0100 Subject: [PATCH 049/127] fuse: Report fuse.Attr.Blocks correctly Fixes #4239. --- changelog/unreleased/issue-4239 | 11 +++++++++++ internal/fuse/file.go | 2 +- internal/fuse/fuse_test.go | 32 ++++++++++++++++++++++++++++++++ internal/fuse/link.go | 2 +- internal/fuse/snapshots_dir.go | 2 +- 5 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 changelog/unreleased/issue-4239 diff --git a/changelog/unreleased/issue-4239 b/changelog/unreleased/issue-4239 new file mode 100644 index 000000000..247f3d9ed --- /dev/null +++ b/changelog/unreleased/issue-4239 @@ -0,0 +1,11 @@ +Bugfix: Correct number of blocks reported in mount point + +Restic mount points incorrectly reported the number of 512-byte (POSIX +standard) blocks for files and links, due to a rounding bug. In particular, +empty files were reported as taking one block instead of zero. + +The rounding is now fixed: the number of blocks reported is the file size +(or link target size), divided by 512 and rounded up to a whole number. + +https://github.com/restic/restic/issues/4239 +https://github.com/restic/restic/pull/4240 diff --git a/internal/fuse/file.go b/internal/fuse/file.go index 28ff5d450..35bc2a73e 100644 --- a/internal/fuse/file.go +++ b/internal/fuse/file.go @@ -50,7 +50,7 @@ func (f *file) Attr(ctx context.Context, a *fuse.Attr) error { a.Inode = f.inode a.Mode = f.node.Mode a.Size = f.node.Size - a.Blocks = (f.node.Size / blockSize) + 1 + a.Blocks = (f.node.Size + blockSize - 1) / blockSize a.BlockSize = blockSize a.Nlink = uint32(f.node.Links) diff --git a/internal/fuse/fuse_test.go b/internal/fuse/fuse_test.go index e71bf6fee..863c7672d 100644 --- a/internal/fuse/fuse_test.go +++ b/internal/fuse/fuse_test.go @@ -8,6 +8,7 @@ import ( "context" "math/rand" "os" + "strings" "testing" "time" @@ -216,6 +217,37 @@ func testTopUIDGID(t *testing.T, cfg Config, repo restic.Repository, uid, gid ui rtest.Equals(t, uint32(0), attr.Gid) } +// Test reporting of fuse.Attr.Blocks in multiples of 512. +func TestBlocks(t *testing.T) { + root := &Root{} + + for _, c := range []struct { + size, blocks uint64 + }{ + {0, 0}, + {1, 1}, + {511, 1}, + {512, 1}, + {513, 2}, + {1024, 2}, + {1025, 3}, + {41253, 81}, + } { + target := strings.Repeat("x", int(c.size)) + + for _, n := range []fs.Node{ + &file{root: root, node: &restic.Node{Size: uint64(c.size)}}, + &link{root: root, node: &restic.Node{LinkTarget: target}}, + &snapshotLink{root: root, snapshot: &restic.Snapshot{}, target: target}, + } { + var a fuse.Attr + err := n.Attr(context.TODO(), &a) + rtest.OK(t, err) + rtest.Equals(t, c.blocks, a.Blocks) + } + } +} + func TestInodeFromNode(t *testing.T) { node := &restic.Node{Name: "foo.txt", Type: "chardev", Links: 2} ino1 := inodeFromNode(1, node) diff --git a/internal/fuse/link.go b/internal/fuse/link.go index f910aadc4..47ee666a3 100644 --- a/internal/fuse/link.go +++ b/internal/fuse/link.go @@ -42,7 +42,7 @@ func (l *link) Attr(ctx context.Context, a *fuse.Attr) error { a.Nlink = uint32(l.node.Links) a.Size = uint64(len(l.node.LinkTarget)) - a.Blocks = 1 + a.Size/blockSize + a.Blocks = (a.Size + blockSize - 1) / blockSize return nil } diff --git a/internal/fuse/snapshots_dir.go b/internal/fuse/snapshots_dir.go index 977d0ab17..c19155741 100644 --- a/internal/fuse/snapshots_dir.go +++ b/internal/fuse/snapshots_dir.go @@ -142,7 +142,7 @@ func (l *snapshotLink) Attr(ctx context.Context, a *fuse.Attr) error { a.Inode = l.inode a.Mode = os.ModeSymlink | 0777 a.Size = uint64(len(l.target)) - a.Blocks = 1 + a.Size/blockSize + a.Blocks = (a.Size + blockSize - 1) / blockSize a.Uid = l.root.uid a.Gid = l.root.gid a.Atime = l.snapshot.Time From 3b24c15c3db794fbd9a79ac8159f109134cc4434 Mon Sep 17 00:00:00 2001 From: greatroar <61184462+greatroar@users.noreply.github.com> Date: Tue, 21 Mar 2023 17:33:18 +0100 Subject: [PATCH 050/127] fuse: Mix inode hashes in a non-symmetric way Since 0.15 (#4020), inodes are generated as hashes of names, xor'd with the parent inode. That means that the inode of a/b/b is h(a/b/b) = h(a) ^ h(b) ^ h(b) = h(a). I.e., the grandchild has the same inode as the grandparent. GNU find trips over this because it thinks it has encountered a loop in the filesystem, and fails to search a/b/b. This happens more generally when the same name occurs an even number of times. Fix this by multiplying the parent by a large prime, so the combining operation is not longer symmetric in its arguments. This is what the FNV hash does, which we used prior to 0.15. The hash is now h(a/b/b) = h(b) ^ p*(h(b) ^ p*h(a)) Note that we already ensure that h(x) is never zero. Collisions can still occur, but they should be much less likely to occur within a single path. Fixes #4253. --- changelog/unreleased/issue-4253 | 18 ++++++++++++++++++ internal/fuse/fuse_test.go | 11 +++++++++++ internal/fuse/inode.go | 6 ++++-- 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/issue-4253 diff --git a/changelog/unreleased/issue-4253 b/changelog/unreleased/issue-4253 new file mode 100644 index 000000000..2471eab0b --- /dev/null +++ b/changelog/unreleased/issue-4253 @@ -0,0 +1,18 @@ +Bugfix: Mount command should no longer create spurious filesystem loops + +When a backup contains a directory that has the same name as its parent, +say, a/b/b, and the GNU find command were run on this backup in a restic +mount, find command would refuse to traverse the lowest "b" directory, +instead printing "File system loop detected". This is due to the way the +restic mount command generates inode numbers for directories in the mount +point. + +The rule for generating these inode numbers was changed in 0.15.0. It has +now been changed again to avoid this issue. A perfect rule does not exist, +but the probability of this behavior occurring is now extremely small. +When it does occur, the mount point is not broken, and scripts that traverse +the mount point should work as long as they don't rely on inode numbers for +detecting filesystem loops. + +https://github.com/restic/restic/issues/4253 +https://github.com/restic/restic/pull/4255 diff --git a/internal/fuse/fuse_test.go b/internal/fuse/fuse_test.go index 863c7672d..9ca1ec0c6 100644 --- a/internal/fuse/fuse_test.go +++ b/internal/fuse/fuse_test.go @@ -258,6 +258,17 @@ func TestInodeFromNode(t *testing.T) { ino1 = inodeFromNode(1, node) ino2 = inodeFromNode(2, node) rtest.Assert(t, ino1 != ino2, "same inode %d but different parent", ino1) + + // Regression test: in a path a/b/b, the grandchild should not get the + // same inode as the grandparent. + a := &restic.Node{Name: "a", Type: "dir", Links: 2} + ab := &restic.Node{Name: "b", Type: "dir", Links: 2} + abb := &restic.Node{Name: "b", Type: "dir", Links: 2} + inoA := inodeFromNode(1, a) + inoAb := inodeFromNode(inoA, ab) + inoAbb := inodeFromNode(inoAb, abb) + rtest.Assert(t, inoA != inoAb, "inode(a/b) = inode(a)") + rtest.Assert(t, inoA != inoAbb, "inode(a/b/b) = inode(a)") } var sink uint64 diff --git a/internal/fuse/inode.go b/internal/fuse/inode.go index de975b167..5e2ece4ac 100644 --- a/internal/fuse/inode.go +++ b/internal/fuse/inode.go @@ -10,9 +10,11 @@ import ( "github.com/restic/restic/internal/restic" ) +const prime = 11400714785074694791 // prime1 from xxhash. + // inodeFromName generates an inode number for a file in a meta dir. func inodeFromName(parent uint64, name string) uint64 { - inode := parent ^ xxhash.Sum64String(cleanupNodeName(name)) + inode := prime*parent ^ xxhash.Sum64String(cleanupNodeName(name)) // Inode 0 is invalid and 1 is the root. Remap those. if inode < 2 { @@ -33,7 +35,7 @@ func inodeFromNode(parent uint64, node *restic.Node) (inode uint64) { } else { // Else, use the name and the parent inode. // node.{DeviceID,Inode} may not even be reliable. - inode = parent ^ xxhash.Sum64String(cleanupNodeName(node.Name)) + inode = prime*parent ^ xxhash.Sum64String(cleanupNodeName(node.Name)) } // Inode 0 is invalid and 1 is the root. Remap those. From fab4a8a4d29089e80e561670094403aa01ee184b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 14 Apr 2023 21:53:01 +0200 Subject: [PATCH 051/127] Properly initialize the --group-by option for backup tests --- cmd/restic/integration_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index d3882116e..93162b223 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -71,6 +71,7 @@ func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts defer cleanup() } + opts.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true} backupErr := runBackup(ctx, opts, gopts, term, target) cancel() From 2841a87cc6e37c961bf6d4b2ca2cc8b888f1a5e9 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 14 Apr 2023 21:53:55 +0200 Subject: [PATCH 052/127] Fix snapshot filtering for relative paths in the backup command The snapshot filtering internally converts relative paths to absolute ones to ensure that the parent snapshots selection works for backups of relative paths. --- internal/restic/snapshot_find.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/restic/snapshot_find.go b/internal/restic/snapshot_find.go index 4d4bb4957..8d6f8c4b1 100644 --- a/internal/restic/snapshot_find.go +++ b/internal/restic/snapshot_find.go @@ -46,6 +46,7 @@ func (f *SnapshotFilter) findLatest(ctx context.Context, be Lister, loader Loade } absTargets = append(absTargets, filepath.Clean(target)) } + f.Paths = absTargets var latest *Snapshot From ba16904eed17e62449b2b415fbfc6b049d6a9088 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 14 Apr 2023 22:21:43 +0200 Subject: [PATCH 053/127] backup: Add test to verify parent snapshot selection for relative paths --- cmd/restic/integration_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 93162b223..dbefe83a7 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -363,6 +363,28 @@ func testBackup(t *testing.T, useFsSnapshot bool) { testRunCheck(t, env.gopts) } +func TestBackupWithRelativePath(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{} + + // first backup + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts) + snapshotIDs := testRunList(t, "snapshots", env.gopts) + rtest.Assert(t, len(snapshotIDs) == 1, "expected one snapshot, got %v", snapshotIDs) + firstSnapshotID := snapshotIDs[0] + + // second backup, implicit incremental + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts) + + // that the correct parent snapshot was used + latestSn, _ := testRunSnapshots(t, env.gopts) + rtest.Assert(t, latestSn != nil, "missing latest snapshot") + rtest.Assert(t, latestSn.Parent != nil && latestSn.Parent.Equal(firstSnapshotID), "second snapshot selected unexpected parent %v instead of %v", latestSn.Parent, firstSnapshotID) +} + func TestDryRunBackup(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() From 8bfc2519d7ce10133d80923cfe7b6f0f5a3794da Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 7 Apr 2023 21:29:01 +0200 Subject: [PATCH 054/127] backend: Deduplicate sanity checks for parameters of Load() method The check is now handled by backend.DefaultLoad. This also guarantees consistent behavior across all backends. --- internal/backend/azure/azure.go | 11 ----------- internal/backend/b2/b2.go | 11 ----------- internal/backend/gs/gs.go | 10 ---------- internal/backend/local/local.go | 7 ------- internal/backend/mem/mem_backend.go | 3 --- internal/backend/s3/s3.go | 11 ----------- internal/backend/sftp/sftp.go | 7 ------- internal/backend/swift/swift.go | 11 ----------- internal/backend/utils.go | 10 ++++++++++ 9 files changed, 10 insertions(+), 71 deletions(-) diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index c92fa3f89..8a252fc81 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -300,17 +300,6 @@ func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h)) - if err := h.Valid(); err != nil { - return nil, backoff.Permanent(err) - } - - if offset < 0 { - return nil, errors.New("offset is negative") - } - - if length < 0 { - return nil, errors.Errorf("invalid length %d", length) - } objName := be.Filename(h) blockBlobClient := be.container.NewBlobClient(objName) diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go index 40dbbf893..b30d1eeab 100644 --- a/internal/backend/b2/b2.go +++ b/internal/backend/b2/b2.go @@ -207,17 +207,6 @@ func (be *b2Backend) Load(ctx context.Context, h restic.Handle, length int, offs func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h)) - if err := h.Valid(); err != nil { - return nil, backoff.Permanent(err) - } - - if offset < 0 { - return nil, errors.New("offset is negative") - } - - if length < 0 { - return nil, errors.Errorf("invalid length %d", length) - } ctx, cancel := context.WithCancel(ctx) diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index 77cbcda97..ee3c30e70 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -273,17 +273,7 @@ func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h)) - if err := h.Valid(); err != nil { - return nil, err - } - if offset < 0 { - return nil, errors.New("offset is negative") - } - - if length < 0 { - return nil, errors.Errorf("invalid length %d", length) - } if length == 0 { // negative length indicates read till end to GCS lib length = -1 diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index 1716e0f07..f514647a6 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -218,13 +218,6 @@ func (b *Local) Load(ctx context.Context, h restic.Handle, length int, offset in func (b *Local) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { debug.Log("Load %v, length %v, offset %v", h, length, offset) - if err := h.Valid(); err != nil { - return nil, backoff.Permanent(err) - } - - if offset < 0 { - return nil, errors.New("offset is negative") - } b.sem.GetToken() f, err := fs.Open(b.Filename(h)) diff --git a/internal/backend/mem/mem_backend.go b/internal/backend/mem/mem_backend.go index 0c46dcd6e..dbdbf1c46 100644 --- a/internal/backend/mem/mem_backend.go +++ b/internal/backend/mem/mem_backend.go @@ -114,9 +114,6 @@ func (be *MemoryBackend) Load(ctx context.Context, h restic.Handle, length int, } func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - if err := h.Valid(); err != nil { - return nil, backoff.Permanent(err) - } be.sem.GetToken() be.m.Lock() diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index ad652a206..91643f909 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -312,17 +312,6 @@ func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h)) - if err := h.Valid(); err != nil { - return nil, backoff.Permanent(err) - } - - if offset < 0 { - return nil, errors.New("offset is negative") - } - - if length < 0 { - return nil, errors.Errorf("invalid length %d", length) - } objName := be.Filename(h) opts := minio.GetObjectOptions{} diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index 514dd58da..5d5aa90d0 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -430,13 +430,6 @@ func (wr *wrapReader) Close() error { func (r *SFTP) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { debug.Log("Load %v, length %v, offset %v", h, length, offset) - if err := h.Valid(); err != nil { - return nil, backoff.Permanent(err) - } - - if offset < 0 { - return nil, errors.New("offset is negative") - } r.sem.GetToken() f, err := r.c.Open(r.Filename(h)) diff --git a/internal/backend/swift/swift.go b/internal/backend/swift/swift.go index 764c7bb62..8685b336f 100644 --- a/internal/backend/swift/swift.go +++ b/internal/backend/swift/swift.go @@ -144,17 +144,6 @@ func (be *beSwift) Load(ctx context.Context, h restic.Handle, length int, offset func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { debug.Log("Load %v, length %v, offset %v", h, length, offset) - if err := h.Valid(); err != nil { - return nil, backoff.Permanent(err) - } - - if offset < 0 { - return nil, errors.New("offset is negative") - } - - if length < 0 { - return nil, errors.Errorf("invalid length %d", length) - } objName := be.Filename(h) diff --git a/internal/backend/utils.go b/internal/backend/utils.go index d2ac44670..1c1607e04 100644 --- a/internal/backend/utils.go +++ b/internal/backend/utils.go @@ -6,6 +6,7 @@ import ( "fmt" "io" + "github.com/cenkalti/backoff/v4" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -62,6 +63,15 @@ func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser { func DefaultLoad(ctx context.Context, h restic.Handle, length int, offset int64, openReader func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error), fn func(rd io.Reader) error) error { + if err := h.Valid(); err != nil { + return backoff.Permanent(err) + } + if offset < 0 { + return errors.New("offset is negative") + } + if length < 0 { + return errors.Errorf("invalid length %d", length) + } rd, err := openReader(ctx, h, length, offset) if err != nil { return err From 4703473ec564ed1130cb87ff4b355fb20babfc99 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 7 Apr 2023 22:01:30 +0200 Subject: [PATCH 055/127] backend: extract most debug logs into logger backend --- cmd/restic/global.go | 33 +++++++---- internal/backend/azure/azure.go | 14 ----- internal/backend/b2/b2.go | 12 ---- internal/backend/dryrun/dry_backend.go | 2 - internal/backend/gs/gs.go | 13 ----- internal/backend/local/local.go | 9 --- internal/backend/logger/log.go | 77 ++++++++++++++++++++++++++ internal/backend/mem/mem_backend.go | 3 - internal/backend/rest/rest.go | 1 - internal/backend/s3/s3.go | 16 ------ internal/backend/sftp/sftp.go | 8 --- internal/backend/swift/swift.go | 11 ---- 12 files changed, 99 insertions(+), 100 deletions(-) create mode 100644 internal/backend/logger/log.go diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 32f18a67f..41f97b5df 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -20,6 +20,7 @@ import ( "github.com/restic/restic/internal/backend/limiter" "github.com/restic/restic/internal/backend/local" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/logger" "github.com/restic/restic/internal/backend/rclone" "github.com/restic/restic/internal/backend/rest" "github.com/restic/restic/internal/backend/retry" @@ -743,6 +744,9 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio return nil, errors.Fatalf("unable to open repository at %v: %v", location.StripPassword(s), err) } + // wrap with debug logging + be = logger.New(be) + // wrap backend if a test specified an inner hook if gopts.backendInnerTestHook != nil { be, err = gopts.backendInnerTestHook(be) @@ -787,27 +791,34 @@ func create(ctx context.Context, s string, opts options.Options) (restic.Backend return nil, err } + var be restic.Backend switch loc.Scheme { case "local": - return local.Create(ctx, cfg.(local.Config)) + be, err = local.Create(ctx, cfg.(local.Config)) case "sftp": - return sftp.Create(ctx, cfg.(sftp.Config)) + be, err = sftp.Create(ctx, cfg.(sftp.Config)) case "s3": - return s3.Create(ctx, cfg.(s3.Config), rt) + be, err = s3.Create(ctx, cfg.(s3.Config), rt) case "gs": - return gs.Create(cfg.(gs.Config), rt) + be, err = gs.Create(cfg.(gs.Config), rt) case "azure": - return azure.Create(ctx, cfg.(azure.Config), rt) + be, err = azure.Create(ctx, cfg.(azure.Config), rt) case "swift": - return swift.Open(ctx, cfg.(swift.Config), rt) + be, err = swift.Open(ctx, cfg.(swift.Config), rt) case "b2": - return b2.Create(ctx, cfg.(b2.Config), rt) + be, err = b2.Create(ctx, cfg.(b2.Config), rt) case "rest": - return rest.Create(ctx, cfg.(rest.Config), rt) + be, err = rest.Create(ctx, cfg.(rest.Config), rt) case "rclone": - return rclone.Create(ctx, cfg.(rclone.Config)) + be, err = rclone.Create(ctx, cfg.(rclone.Config)) + default: + debug.Log("invalid repository scheme: %v", s) + return nil, errors.Fatalf("invalid scheme %q", loc.Scheme) } - debug.Log("invalid repository scheme: %v", s) - return nil, errors.Fatalf("invalid scheme %q", loc.Scheme) + if err != nil { + return nil, err + } + + return logger.New(be), nil } diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index 8a252fc81..4d7a4a57b 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -152,7 +152,6 @@ func (be *Backend) SetListMaxItems(i int) { // IsNotExist returns true if the error is caused by a not existing file. func (be *Backend) IsNotExist(err error) bool { - debug.Log("IsNotExist(%T, %#v)", err, err) return bloberror.HasCode(err, bloberror.BlobNotFound) } @@ -193,8 +192,6 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe objName := be.Filename(h) - debug.Log("Save %v at %v", h, objName) - be.sem.GetToken() debug.Log("InsertObject(%v, %v)", be.cfg.AccountName, objName) @@ -209,8 +206,6 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe } be.sem.ReleaseToken() - debug.Log("%v, err %#v", objName, err) - return err } @@ -299,8 +294,6 @@ func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset } func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h)) - objName := be.Filename(h) blockBlobClient := be.container.NewBlobClient(objName) @@ -322,8 +315,6 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, // Stat returns information about a blob. func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { - debug.Log("%v", h) - objName := be.Filename(h) blobClient := be.container.NewBlobClient(objName) @@ -332,7 +323,6 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, be.sem.ReleaseToken() if err != nil { - debug.Log("blob.GetProperties err %v", err) return restic.FileInfo{}, errors.Wrap(err, "blob.GetProperties") } @@ -352,8 +342,6 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { _, err := blob.Delete(ctx, &azblob.DeleteBlobOptions{}) be.sem.ReleaseToken() - debug.Log("Remove(%v) at %v -> err %v", h, objName, err) - if be.IsNotExist(err) { return nil } @@ -364,8 +352,6 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - debug.Log("listing %v", t) - prefix, _ := be.Basedir(t) // make sure prefix ends with a slash diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go index b30d1eeab..10f1a715b 100644 --- a/internal/backend/b2/b2.go +++ b/internal/backend/b2/b2.go @@ -206,8 +206,6 @@ func (be *b2Backend) Load(ctx context.Context, h restic.Handle, length int, offs } func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h)) - ctx, cancel := context.WithCancel(ctx) be.sem.GetToken() @@ -249,7 +247,6 @@ func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.Rewind // b2 always requires sha1 checksums for uploaded file parts w := obj.NewWriter(ctx) n, err := io.Copy(w, rd) - debug.Log(" saved %d bytes, err %v", n, err) if err != nil { _ = w.Close() @@ -265,8 +262,6 @@ func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.Rewind // Stat returns information about a blob. func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { - debug.Log("Stat %v", h) - be.sem.GetToken() defer be.sem.ReleaseToken() @@ -274,7 +269,6 @@ func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileI obj := be.bucket.Object(name) info, err := obj.Attrs(ctx) if err != nil { - debug.Log("Attrs() err %v", err) return restic.FileInfo{}, errors.Wrap(err, "Stat") } return restic.FileInfo{Size: info.Size, Name: h.Name}, nil @@ -282,8 +276,6 @@ func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileI // Remove removes the blob with the given name and type. func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error { - debug.Log("Remove %v", h) - be.sem.GetToken() defer be.sem.ReleaseToken() @@ -330,8 +322,6 @@ func (sm *semLocker) Unlock() { sm.ReleaseToken() } // List returns a channel that yields all names of blobs of type t. func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - debug.Log("List %v", t) - ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -356,7 +346,6 @@ func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic } } if err := iter.Err(); err != nil { - debug.Log("List: %v", err) return err } return nil @@ -364,7 +353,6 @@ func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic // Remove keys for a specified backend type. func (be *b2Backend) removeKeys(ctx context.Context, t restic.FileType) error { - debug.Log("removeKeys %v", t) return be.List(ctx, t, func(fi restic.FileInfo) error { return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) }) diff --git a/internal/backend/dryrun/dry_backend.go b/internal/backend/dryrun/dry_backend.go index 37569c320..1218e9819 100644 --- a/internal/backend/dryrun/dry_backend.go +++ b/internal/backend/dryrun/dry_backend.go @@ -34,8 +34,6 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe return err } - debug.Log("faked saving %v bytes at %v", rd.Length(), h) - // don't save anything, just return ok return nil } diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index ee3c30e70..faf8b9858 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -169,7 +169,6 @@ func (be *Backend) SetListMaxItems(i int) { // IsNotExist returns true if the error is caused by a not existing file. func (be *Backend) IsNotExist(err error) bool { - debug.Log("IsNotExist(%T, %#v)", err, err) return errors.Is(err, storage.ErrObjectNotExist) } @@ -210,8 +209,6 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe objName := be.Filename(h) - debug.Log("Save %v at %v", h, objName) - be.sem.GetToken() debug.Log("InsertObject(%v, %v)", be.bucketName, objName) @@ -253,11 +250,9 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe be.sem.ReleaseToken() if err != nil { - debug.Log("%v: err %#v: %v", objName, err, err) return errors.Wrap(err, "service.Objects.Insert") } - debug.Log("%v -> %v bytes", objName, wbytes) // sanity check if wbytes != rd.Length() { return errors.Errorf("wrote %d bytes instead of the expected %d bytes", wbytes, rd.Length()) @@ -272,8 +267,6 @@ func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset } func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h)) - if length == 0 { // negative length indicates read till end to GCS lib length = -1 @@ -297,8 +290,6 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, // Stat returns information about a blob. func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { - debug.Log("%v", h) - objName := be.Filename(h) be.sem.GetToken() @@ -306,7 +297,6 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf be.sem.ReleaseToken() if err != nil { - debug.Log("GetObjectAttributes() err %v", err) return restic.FileInfo{}, errors.Wrap(err, "service.Objects.Get") } @@ -325,15 +315,12 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { err = nil } - debug.Log("Remove(%v) at %v -> err %v", h, objName, err) return errors.Wrap(err, "client.RemoveObject") } // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - debug.Log("listing %v", t) - prefix, _ := be.Basedir(t) // make sure prefix ends with a slash diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index f514647a6..a1f3c6091 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -114,7 +114,6 @@ func (b *Local) IsNotExist(err error) bool { // Save stores data in the backend at the handle. func (b *Local) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) (err error) { - debug.Log("Save %v", h) if err := h.Valid(); err != nil { return backoff.Permanent(err) } @@ -217,8 +216,6 @@ func (b *Local) Load(ctx context.Context, h restic.Handle, length int, offset in } func (b *Local) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - debug.Log("Load %v, length %v, offset %v", h, length, offset) - b.sem.GetToken() f, err := fs.Open(b.Filename(h)) if err != nil { @@ -246,7 +243,6 @@ func (b *Local) openReader(ctx context.Context, h restic.Handle, length int, off // Stat returns information about a blob. func (b *Local) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { - debug.Log("Stat %v", h) if err := h.Valid(); err != nil { return restic.FileInfo{}, backoff.Permanent(err) } @@ -264,7 +260,6 @@ func (b *Local) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, err // Remove removes the blob with the given name and type. func (b *Local) Remove(ctx context.Context, h restic.Handle) error { - debug.Log("Remove %v", h) fn := b.Filename(h) b.sem.GetToken() @@ -282,8 +277,6 @@ func (b *Local) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) (err error) { - debug.Log("List %v", t) - basedir, subdirs := b.Basedir(t) if subdirs { err = visitDirs(ctx, basedir, fn) @@ -377,13 +370,11 @@ func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error, // Delete removes the repository and all files. func (b *Local) Delete(ctx context.Context) error { - debug.Log("Delete()") return fs.RemoveAll(b.Path) } // Close closes all open files. func (b *Local) Close() error { - debug.Log("Close()") // this does not need to do anything, all open files are closed within the // same function. return nil diff --git a/internal/backend/logger/log.go b/internal/backend/logger/log.go new file mode 100644 index 000000000..4623d8021 --- /dev/null +++ b/internal/backend/logger/log.go @@ -0,0 +1,77 @@ +package logger + +import ( + "context" + "io" + + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/restic" +) + +type Backend struct { + restic.Backend +} + +// statically ensure that Backend implements restic.Backend. +var _ restic.Backend = &Backend{} + +func New(be restic.Backend) *Backend { + return &Backend{Backend: be} +} + +func (be *Backend) IsNotExist(err error) bool { + isNotExist := be.Backend.IsNotExist(err) + debug.Log("IsNotExist(%T, %#v, %v)", err, err, isNotExist) + return isNotExist +} + +// Save adds new Data to the backend. +func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + debug.Log("Save(%v, %v)", h, rd.Length()) + err := be.Backend.Save(ctx, h, rd) + debug.Log(" save err %v", err) + return err +} + +// Remove deletes a file from the backend. +func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { + debug.Log("Remove(%v)", h) + err := be.Backend.Remove(ctx, h) + debug.Log(" remove err %v", err) + return err +} + +func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(io.Reader) error) error { + debug.Log("Load(%v, length %v, offset %v)", h, length, offset) + err := be.Backend.Load(ctx, h, length, offset, fn) + debug.Log(" load err %v", err) + return err +} + +func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { + debug.Log("Stat(%v)", h) + fi, err := be.Backend.Stat(ctx, h) + debug.Log(" stat err %v", err) + return fi, err +} + +func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { + debug.Log("List(%v)", t) + err := be.Backend.List(ctx, t, fn) + debug.Log(" list err %v", err) + return err +} + +func (be *Backend) Delete(ctx context.Context) error { + debug.Log("Delete()") + err := be.Backend.Delete(ctx) + debug.Log(" delete err %v", err) + return err +} + +func (be *Backend) Close() error { + debug.Log("Close()") + err := be.Backend.Close() + debug.Log(" close err %v", err) + return err +} diff --git a/internal/backend/mem/mem_backend.go b/internal/backend/mem/mem_backend.go index dbdbf1c46..59e89286e 100644 --- a/internal/backend/mem/mem_backend.go +++ b/internal/backend/mem/mem_backend.go @@ -102,7 +102,6 @@ func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re } be.data[h] = buf - debug.Log("saved %v bytes at %v", len(buf), h) return ctx.Err() } @@ -167,8 +166,6 @@ func (be *MemoryBackend) Stat(ctx context.Context, h restic.Handle) (restic.File h.Name = "" } - debug.Log("stat %v", h) - e, ok := be.data[h] if !ok { return restic.FileInfo{}, errNotFound diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index f4c2897b9..ad5af1629 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -212,7 +212,6 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset } func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - debug.Log("Load %v, length %v, offset %v", h, length, offset) if err := h.Valid(); err != nil { return nil, backoff.Permanent(err) } diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index 91643f909..872fb0441 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -169,8 +169,6 @@ func isAccessDenied(err error) bool { // IsNotExist returns true if the error is caused by a not existing file. func (be *Backend) IsNotExist(err error) bool { - debug.Log("IsNotExist(%T, %#v)", err, err) - var e minio.ErrorResponse return errors.As(err, &e) && e.Code == "NoSuchKey" } @@ -273,8 +271,6 @@ func (be *Backend) Path() string { // Save stores data in the backend at the handle. func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { - debug.Log("Save %v", h) - if err := h.Valid(); err != nil { return backoff.Permanent(err) } @@ -294,8 +290,6 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe debug.Log("PutObject(%v, %v, %v)", be.cfg.Bucket, objName, rd.Length()) info, err := be.client.PutObject(ctx, be.cfg.Bucket, objName, io.NopCloser(rd), int64(rd.Length()), opts) - debug.Log("%v -> %v bytes, err %#v: %v", objName, info.Size, err, err) - // sanity check if err == nil && info.Size != rd.Length() { return errors.Errorf("wrote %d bytes instead of the expected %d bytes", info.Size, rd.Length()) @@ -311,8 +305,6 @@ func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset } func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h)) - objName := be.Filename(h) opts := minio.GetObjectOptions{} @@ -345,8 +337,6 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, // Stat returns information about a blob. func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { - debug.Log("%v", h) - objName := be.Filename(h) var obj *minio.Object @@ -355,7 +345,6 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf be.sem.GetToken() obj, err = be.client.GetObject(ctx, be.cfg.Bucket, objName, opts) if err != nil { - debug.Log("GetObject() err %v", err) be.sem.ReleaseToken() return restic.FileInfo{}, errors.Wrap(err, "client.GetObject") } @@ -371,7 +360,6 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf fi, err := obj.Stat() if err != nil { - debug.Log("Stat() err %v", err) return restic.FileInfo{}, errors.Wrap(err, "Stat") } @@ -386,8 +374,6 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { err := be.client.RemoveObject(ctx, be.cfg.Bucket, objName, minio.RemoveObjectOptions{}) be.sem.ReleaseToken() - debug.Log("Remove(%v) at %v -> err %v", h, objName, err) - if be.IsNotExist(err) { err = nil } @@ -398,8 +384,6 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - debug.Log("listing %v", t) - prefix, recursive := be.Basedir(t) // make sure prefix ends with a slash diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index 5d5aa90d0..afe3fc394 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -304,7 +304,6 @@ func tempSuffix() string { // Save stores data in the backend at the handle. func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { - debug.Log("Save %v", h) if err := r.clientError(); err != nil { return err } @@ -429,8 +428,6 @@ func (wr *wrapReader) Close() error { } func (r *SFTP) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - debug.Log("Load %v, length %v, offset %v", h, length, offset) - r.sem.GetToken() f, err := r.c.Open(r.Filename(h)) if err != nil { @@ -467,7 +464,6 @@ func (r *SFTP) openReader(ctx context.Context, h restic.Handle, length int, offs // Stat returns information about a blob. func (r *SFTP) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { - debug.Log("Stat(%v)", h) if err := r.clientError(); err != nil { return restic.FileInfo{}, err } @@ -489,7 +485,6 @@ func (r *SFTP) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, erro // Remove removes the content stored at name. func (r *SFTP) Remove(ctx context.Context, h restic.Handle) error { - debug.Log("Remove(%v)", h) if err := r.clientError(); err != nil { return err } @@ -503,8 +498,6 @@ func (r *SFTP) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. func (r *SFTP) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - debug.Log("List %v", t) - basedir, subdirs := r.Basedir(t) walker := r.c.Walk(basedir) for { @@ -565,7 +558,6 @@ var closeTimeout = 2 * time.Second // Close closes the sftp connection and terminates the underlying command. func (r *SFTP) Close() error { - debug.Log("Close") if r == nil { return nil } diff --git a/internal/backend/swift/swift.go b/internal/backend/swift/swift.go index 8685b336f..99940df5c 100644 --- a/internal/backend/swift/swift.go +++ b/internal/backend/swift/swift.go @@ -143,7 +143,6 @@ func (be *beSwift) Load(ctx context.Context, h restic.Handle, length int, offset } func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - debug.Log("Load %v, length %v, offset %v", h, length, offset) objName := be.Filename(h) @@ -163,7 +162,6 @@ func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int, be.sem.GetToken() obj, _, err := be.conn.ObjectOpen(ctx, be.container, objName, false, headers) if err != nil { - debug.Log(" err %v", err) be.sem.ReleaseToken() return nil, errors.Wrap(err, "conn.ObjectOpen") } @@ -179,8 +177,6 @@ func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe objName := be.Filename(h) - debug.Log("Save %v at %v", h, objName) - be.sem.GetToken() defer be.sem.ReleaseToken() @@ -192,15 +188,12 @@ func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe be.container, objName, rd, true, hex.EncodeToString(rd.Hash()), encoding, hdr) // swift does not return the upload length - debug.Log("%v, err %#v", objName, err) return errors.Wrap(err, "client.PutObject") } // Stat returns information about a blob. func (be *beSwift) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { - debug.Log("%v", h) - objName := be.Filename(h) be.sem.GetToken() @@ -208,7 +201,6 @@ func (be *beSwift) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf obj, _, err := be.conn.Object(ctx, be.container, objName) if err != nil { - debug.Log("Object() err %v", err) return restic.FileInfo{}, errors.Wrap(err, "conn.Object") } @@ -223,15 +215,12 @@ func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error { defer be.sem.ReleaseToken() err := be.conn.ObjectDelete(ctx, be.container, objName) - debug.Log("Remove(%v) -> err %v", h, err) return errors.Wrap(err, "conn.ObjectDelete") } // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. func (be *beSwift) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - debug.Log("listing %v", t) - prefix, _ := be.Basedir(t) prefix += "/" From 8b5ab5b59fbb0f17853e1bb303d851ffcdf987ef Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 7 Apr 2023 22:13:32 +0200 Subject: [PATCH 056/127] dryrun: fix outdated comments --- internal/backend/dryrun/dry_backend.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/backend/dryrun/dry_backend.go b/internal/backend/dryrun/dry_backend.go index 1218e9819..487e2bc33 100644 --- a/internal/backend/dryrun/dry_backend.go +++ b/internal/backend/dryrun/dry_backend.go @@ -18,10 +18,9 @@ type Backend struct { b restic.Backend } -// statically ensure that RetryBackend implements restic.Backend. +// statically ensure that Backend implements restic.Backend. var _ restic.Backend = &Backend{} -// New returns a new backend that saves all data in a map in memory. func New(be restic.Backend) *Backend { b := &Backend{b: be} debug.Log("created new dry backend") From 8e1e3844aa6a1559449c5a5b2369b522817eb109 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 7 Apr 2023 23:02:35 +0200 Subject: [PATCH 057/127] backend: factor out connection limiting and parameter validation The SemaphoreBackend now uniformly enforces the limit of concurrent backend operations. In addition, it unifies the parameter validation. The List() methods no longer uses a semaphore. Restic already never runs multiple list operations in parallel. By managing the semaphore in a wrapper backend, the sections that hold a semaphore token grow slightly. However, the main bottleneck is IO, so this shouldn't make much of a difference. The key insight that enables the SemaphoreBackend is that all of the complex semaphore handling in `openReader()` still happens within the original call to `Load()`. Thus, getting and releasing the semaphore tokens can be refactored to happen directly in `Load()`. This eliminates the need for wrapping the reader in `openReader()` to release the token. --- cmd/restic/global.go | 7 ++- internal/backend/azure/azure.go | 26 +-------- internal/backend/b2/b2.go | 51 ++--------------- internal/backend/gs/gs.go | 35 ++---------- internal/backend/local/local.go | 34 +---------- internal/backend/mem/mem_backend.go | 40 +------------ internal/backend/rest/rest.go | 44 --------------- internal/backend/s3/s3.go | 31 ++-------- internal/backend/sema/backend.go | 87 +++++++++++++++++++++++++++++ internal/backend/sema/semaphore.go | 58 ++++--------------- internal/backend/sftp/sftp.go | 57 +------------------ internal/backend/swift/swift.go | 29 +--------- internal/backend/utils.go | 11 +--- 13 files changed, 126 insertions(+), 384 deletions(-) create mode 100644 internal/backend/sema/backend.go diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 41f97b5df..8d34f8ddb 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -25,6 +25,7 @@ import ( "github.com/restic/restic/internal/backend/rest" "github.com/restic/restic/internal/backend/retry" "github.com/restic/restic/internal/backend/s3" + "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/backend/sftp" "github.com/restic/restic/internal/backend/swift" "github.com/restic/restic/internal/cache" @@ -744,8 +745,8 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio return nil, errors.Fatalf("unable to open repository at %v: %v", location.StripPassword(s), err) } - // wrap with debug logging - be = logger.New(be) + // wrap with debug logging and connection limiting + be = logger.New(sema.New(be)) // wrap backend if a test specified an inner hook if gopts.backendInnerTestHook != nil { @@ -820,5 +821,5 @@ func create(ctx context.Context, s string, opts options.Options) (restic.Backend return nil, err } - return logger.New(be), nil + return logger.New(sema.New(be)), nil } diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index 4d7a4a57b..82d55960f 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -14,7 +14,6 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" - "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -26,7 +25,6 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" azContainer "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" - "github.com/cenkalti/backoff/v4" ) // Backend stores data on an azure endpoint. @@ -34,7 +32,6 @@ type Backend struct { cfg Config container *azContainer.Client connections uint - sem sema.Semaphore prefix string listMaxItems int layout.Layout @@ -96,16 +93,10 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { return nil, errors.New("no azure authentication information found") } - sem, err := sema.New(cfg.Connections) - if err != nil { - return nil, err - } - be := &Backend{ container: client, cfg: cfg, connections: cfg.Connections, - sem: sem, Layout: &layout.DefaultLayout{ Path: cfg.Prefix, Join: path.Join, @@ -186,14 +177,8 @@ func (be *Backend) Path() string { // Save stores data in the backend at the handle. func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { - if err := h.Valid(); err != nil { - return backoff.Permanent(err) - } - objName := be.Filename(h) - be.sem.GetToken() - debug.Log("InsertObject(%v, %v)", be.cfg.AccountName, objName) var err error @@ -205,7 +190,6 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe err = be.saveLarge(ctx, objName, rd) } - be.sem.ReleaseToken() return err } @@ -297,7 +281,6 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, objName := be.Filename(h) blockBlobClient := be.container.NewBlobClient(objName) - be.sem.GetToken() resp, err := blockBlobClient.DownloadStream(ctx, &blob.DownloadStreamOptions{ Range: azblob.HTTPRange{ Offset: offset, @@ -306,11 +289,10 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, }) if err != nil { - be.sem.ReleaseToken() return nil, err } - return be.sem.ReleaseTokenOnClose(resp.Body, nil), err + return resp.Body, err } // Stat returns information about a blob. @@ -318,9 +300,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, objName := be.Filename(h) blobClient := be.container.NewBlobClient(objName) - be.sem.GetToken() props, err := blobClient.GetProperties(ctx, nil) - be.sem.ReleaseToken() if err != nil { return restic.FileInfo{}, errors.Wrap(err, "blob.GetProperties") @@ -338,9 +318,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { objName := be.Filename(h) blob := be.container.NewBlobClient(objName) - be.sem.GetToken() _, err := blob.Delete(ctx, &azblob.DeleteBlobOptions{}) - be.sem.ReleaseToken() if be.IsNotExist(err) { return nil @@ -368,9 +346,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F lister := be.container.NewListBlobsFlatPager(opts) for lister.More() { - be.sem.GetToken() resp, err := lister.NextPage(ctx) - be.sem.ReleaseToken() if err != nil { return err diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go index 10f1a715b..0827f727b 100644 --- a/internal/backend/b2/b2.go +++ b/internal/backend/b2/b2.go @@ -11,12 +11,10 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" - "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" - "github.com/cenkalti/backoff/v4" "github.com/kurin/blazer/b2" "github.com/kurin/blazer/base" ) @@ -28,7 +26,6 @@ type b2Backend struct { cfg Config listMaxItems int layout.Layout - sem sema.Semaphore canDelete bool } @@ -92,11 +89,6 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend return nil, errors.Wrap(err, "Bucket") } - sem, err := sema.New(cfg.Connections) - if err != nil { - return nil, err - } - be := &b2Backend{ client: client, bucket: bucket, @@ -106,7 +98,6 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend Path: cfg.Prefix, }, listMaxItems: defaultListMaxItems, - sem: sem, canDelete: true, } @@ -134,11 +125,6 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backe return nil, errors.Wrap(err, "NewBucket") } - sem, err := sema.New(cfg.Connections) - if err != nil { - return nil, err - } - be := &b2Backend{ client: client, bucket: bucket, @@ -148,7 +134,6 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backe Path: cfg.Prefix, }, listMaxItems: defaultListMaxItems, - sem: sem, } _, err = be.Stat(ctx, restic.Handle{Type: restic.ConfigFile}) @@ -202,20 +187,18 @@ func (be *b2Backend) IsNotExist(err error) bool { // Load runs fn with a reader that yields the contents of the file at h at the // given offset. func (be *b2Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - ctx, cancel := context.WithCancel(ctx) - - be.sem.GetToken() - name := be.Layout.Filename(h) obj := be.bucket.Object(name) if offset == 0 && length == 0 { - rd := obj.NewReader(ctx) - return be.sem.ReleaseTokenOnClose(rd, cancel), nil + return obj.NewReader(ctx), nil } // pass a negative length to NewRangeReader so that the remainder of the @@ -224,8 +207,7 @@ func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int length = -1 } - rd := obj.NewRangeReader(ctx, offset, int64(length)) - return be.sem.ReleaseTokenOnClose(rd, cancel), nil + return obj.NewRangeReader(ctx, offset, int64(length)), nil } // Save stores data in the backend at the handle. @@ -233,15 +215,7 @@ func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.Rewind ctx, cancel := context.WithCancel(ctx) defer cancel() - if err := h.Valid(); err != nil { - return backoff.Permanent(err) - } - - be.sem.GetToken() - defer be.sem.ReleaseToken() - name := be.Filename(h) - debug.Log("Save %v, name %v", h, name) obj := be.bucket.Object(name) // b2 always requires sha1 checksums for uploaded file parts @@ -262,9 +236,6 @@ func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.Rewind // Stat returns information about a blob. func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { - be.sem.GetToken() - defer be.sem.ReleaseToken() - name := be.Filename(h) obj := be.bucket.Object(name) info, err := obj.Attrs(ctx) @@ -276,9 +247,6 @@ func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileI // Remove removes the blob with the given name and type. func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error { - be.sem.GetToken() - defer be.sem.ReleaseToken() - // the retry backend will also repeat the remove method up to 10 times for i := 0; i < 3; i++ { obj := be.bucket.Object(be.Filename(h)) @@ -313,20 +281,13 @@ func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error { return errors.New("failed to delete all file versions") } -type semLocker struct { - sema.Semaphore -} - -func (sm *semLocker) Lock() { sm.GetToken() } -func (sm *semLocker) Unlock() { sm.ReleaseToken() } - // List returns a channel that yields all names of blobs of type t. func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { ctx, cancel := context.WithCancel(ctx) defer cancel() prefix, _ := be.Basedir(t) - iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(be.listMaxItems), b2.ListLocker(&semLocker{be.sem})) + iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(be.listMaxItems)) for iter.Next() { obj := iter.Object() diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index faf8b9858..12458a79c 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -15,7 +15,6 @@ import ( "github.com/pkg/errors" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" - "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" @@ -37,7 +36,6 @@ type Backend struct { gcsClient *storage.Client projectID string connections uint - sem sema.Semaphore bucketName string bucket *storage.BucketHandle prefix string @@ -99,16 +97,10 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { return nil, errors.Wrap(err, "getStorageClient") } - sem, err := sema.New(cfg.Connections) - if err != nil { - return nil, err - } - be := &Backend{ gcsClient: gcsClient, projectID: cfg.ProjectID, connections: cfg.Connections, - sem: sem, bucketName: cfg.Bucket, bucket: gcsClient.Bucket(cfg.Bucket), prefix: cfg.Prefix, @@ -203,16 +195,8 @@ func (be *Backend) Path() string { // Save stores data in the backend at the handle. func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { - if err := h.Valid(); err != nil { - return err - } - objName := be.Filename(h) - be.sem.GetToken() - - debug.Log("InsertObject(%v, %v)", be.bucketName, objName) - // Set chunk size to zero to disable resumable uploads. // // With a non-zero chunk size (the default is @@ -247,8 +231,6 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe err = cerr } - be.sem.ReleaseToken() - if err != nil { return errors.Wrap(err, "service.Objects.Insert") } @@ -263,6 +245,9 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe // Load runs fn with a reader that yields the contents of the file at h at the // given offset. func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } @@ -274,27 +259,19 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, objName := be.Filename(h) - be.sem.GetToken() - - ctx, cancel := context.WithCancel(ctx) - r, err := be.bucket.Object(objName).NewRangeReader(ctx, offset, int64(length)) if err != nil { - cancel() - be.sem.ReleaseToken() return nil, err } - return be.sem.ReleaseTokenOnClose(r, cancel), err + return r, err } // Stat returns information about a blob. func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { objName := be.Filename(h) - be.sem.GetToken() attr, err := be.bucket.Object(objName).Attrs(ctx) - be.sem.ReleaseToken() if err != nil { return restic.FileInfo{}, errors.Wrap(err, "service.Objects.Get") @@ -307,9 +284,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { objName := be.Filename(h) - be.sem.GetToken() err := be.bucket.Object(objName).Delete(ctx) - be.sem.ReleaseToken() if err == storage.ErrObjectNotExist { err = nil @@ -334,9 +309,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F itr := be.bucket.Objects(ctx, &storage.Query{Prefix: prefix}) for { - be.sem.GetToken() attrs, err := itr.Next() - be.sem.ReleaseToken() if err == iterator.Done { break } diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index a1f3c6091..ca806f754 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -10,7 +10,6 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" - "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" @@ -22,7 +21,6 @@ import ( // Local is a backend in a local directory. type Local struct { Config - sem sema.Semaphore layout.Layout backend.Modes } @@ -38,11 +36,6 @@ func open(ctx context.Context, cfg Config) (*Local, error) { return nil, err } - sem, err := sema.New(cfg.Connections) - if err != nil { - return nil, err - } - fi, err := fs.Stat(l.Filename(restic.Handle{Type: restic.ConfigFile})) m := backend.DeriveModesFromFileInfo(fi, err) debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir) @@ -50,7 +43,6 @@ func open(ctx context.Context, cfg Config) (*Local, error) { return &Local{ Config: cfg, Layout: l, - sem: sem, Modes: m, }, nil } @@ -114,10 +106,6 @@ func (b *Local) IsNotExist(err error) bool { // Save stores data in the backend at the handle. func (b *Local) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) (err error) { - if err := h.Valid(); err != nil { - return backoff.Permanent(err) - } - finalname := b.Filename(h) dir := filepath.Dir(finalname) @@ -128,9 +116,6 @@ func (b *Local) Save(ctx context.Context, h restic.Handle, rd restic.RewindReade } }() - b.sem.GetToken() - defer b.sem.ReleaseToken() - // Create new file with a temporary name. tmpname := filepath.Base(finalname) + "-tmp-" f, err := tempFile(dir, tmpname) @@ -216,40 +201,28 @@ func (b *Local) Load(ctx context.Context, h restic.Handle, length int, offset in } func (b *Local) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - b.sem.GetToken() f, err := fs.Open(b.Filename(h)) if err != nil { - b.sem.ReleaseToken() return nil, err } if offset > 0 { _, err = f.Seek(offset, 0) if err != nil { - b.sem.ReleaseToken() _ = f.Close() return nil, err } } - r := b.sem.ReleaseTokenOnClose(f, nil) - if length > 0 { - return backend.LimitReadCloser(r, int64(length)), nil + return backend.LimitReadCloser(f, int64(length)), nil } - return r, nil + return f, nil } // Stat returns information about a blob. func (b *Local) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { - if err := h.Valid(); err != nil { - return restic.FileInfo{}, backoff.Permanent(err) - } - - b.sem.GetToken() - defer b.sem.ReleaseToken() - fi, err := fs.Stat(b.Filename(h)) if err != nil { return restic.FileInfo{}, errors.WithStack(err) @@ -262,9 +235,6 @@ func (b *Local) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, err func (b *Local) Remove(ctx context.Context, h restic.Handle) error { fn := b.Filename(h) - b.sem.GetToken() - defer b.sem.ReleaseToken() - // reset read-only flag err := fs.Chmod(fn, 0666) if err != nil && !os.IsPermission(err) { diff --git a/internal/backend/mem/mem_backend.go b/internal/backend/mem/mem_backend.go index 59e89286e..4db4c9821 100644 --- a/internal/backend/mem/mem_backend.go +++ b/internal/backend/mem/mem_backend.go @@ -10,12 +10,9 @@ import ( "github.com/cespare/xxhash/v2" "github.com/restic/restic/internal/backend" - "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" - - "github.com/cenkalti/backoff/v4" ) type memMap map[restic.Handle][]byte @@ -32,19 +29,12 @@ const connectionCount = 2 type MemoryBackend struct { data memMap m sync.Mutex - sem sema.Semaphore } // New returns a new backend that saves all data in a map in memory. func New() *MemoryBackend { - sem, err := sema.New(connectionCount) - if err != nil { - panic(err) - } - be := &MemoryBackend{ data: make(memMap), - sem: sem, } debug.Log("created new memory backend") @@ -59,13 +49,6 @@ func (be *MemoryBackend) IsNotExist(err error) bool { // Save adds new Data to the backend. func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { - if err := h.Valid(); err != nil { - return backoff.Permanent(err) - } - - be.sem.GetToken() - defer be.sem.ReleaseToken() - be.m.Lock() defer be.m.Unlock() @@ -113,8 +96,6 @@ func (be *MemoryBackend) Load(ctx context.Context, h restic.Handle, length int, } func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - - be.sem.GetToken() be.m.Lock() defer be.m.Unlock() @@ -123,21 +104,12 @@ func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length h.Name = "" } - debug.Log("Load %v offset %v len %v", h, offset, length) - - if offset < 0 { - be.sem.ReleaseToken() - return nil, errors.New("offset is negative") - } - if _, ok := be.data[h]; !ok { - be.sem.ReleaseToken() return nil, errNotFound } buf := be.data[h] if offset > int64(len(buf)) { - be.sem.ReleaseToken() return nil, errors.New("offset beyond end of file") } @@ -146,18 +118,11 @@ func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length buf = buf[:length] } - return be.sem.ReleaseTokenOnClose(io.NopCloser(bytes.NewReader(buf)), nil), ctx.Err() + return io.NopCloser(bytes.NewReader(buf)), ctx.Err() } // Stat returns information about a file in the backend. func (be *MemoryBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { - if err := h.Valid(); err != nil { - return restic.FileInfo{}, backoff.Permanent(err) - } - - be.sem.GetToken() - defer be.sem.ReleaseToken() - be.m.Lock() defer be.m.Unlock() @@ -176,9 +141,6 @@ func (be *MemoryBackend) Stat(ctx context.Context, h restic.Handle) (restic.File // Remove deletes a file from the backend. func (be *MemoryBackend) Remove(ctx context.Context, h restic.Handle) error { - be.sem.GetToken() - defer be.sem.ReleaseToken() - be.m.Lock() defer be.m.Unlock() diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index ad5af1629..a88e26daa 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -12,12 +12,9 @@ import ( "strings" "github.com/restic/restic/internal/backend/layout" - "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" - - "github.com/cenkalti/backoff/v4" ) // make sure the rest backend implements restic.Backend @@ -27,7 +24,6 @@ var _ restic.Backend = &Backend{} type Backend struct { url *url.URL connections uint - sem sema.Semaphore client http.Client layout.Layout } @@ -40,11 +36,6 @@ const ( // Open opens the REST backend with the given config. func Open(cfg Config, rt http.RoundTripper) (*Backend, error) { - sem, err := sema.New(cfg.Connections) - if err != nil { - return nil, err - } - // use url without trailing slash for layout url := cfg.URL.String() if url[len(url)-1] == '/' { @@ -56,7 +47,6 @@ func Open(cfg Config, rt http.RoundTripper) (*Backend, error) { client: http.Client{Transport: rt}, Layout: &layout.RESTLayout{URL: url, Join: path.Join}, connections: cfg.Connections, - sem: sem, } return be, nil @@ -123,10 +113,6 @@ func (b *Backend) HasAtomicReplace() bool { // Save stores data in the backend at the handle. func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { - if err := h.Valid(); err != nil { - return backoff.Permanent(err) - } - ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -143,9 +129,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea // let's the server know what's coming. req.ContentLength = rd.Length() - b.sem.GetToken() resp, err := b.client.Do(req) - b.sem.ReleaseToken() var cerr error if resp != nil { @@ -212,18 +196,6 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset } func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - if err := h.Valid(); err != nil { - return nil, backoff.Permanent(err) - } - - if offset < 0 { - return nil, errors.New("offset is negative") - } - - if length < 0 { - return nil, errors.Errorf("invalid length %d", length) - } - req, err := http.NewRequestWithContext(ctx, "GET", b.Filename(h), nil) if err != nil { return nil, errors.WithStack(err) @@ -237,9 +209,7 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o req.Header.Set("Accept", ContentTypeV2) debug.Log("Load(%v) send range %v", h, byteRange) - b.sem.GetToken() resp, err := b.client.Do(req) - b.sem.ReleaseToken() if err != nil { if resp != nil { @@ -264,19 +234,13 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o // Stat returns information about a blob. func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { - if err := h.Valid(); err != nil { - return restic.FileInfo{}, backoff.Permanent(err) - } - req, err := http.NewRequestWithContext(ctx, http.MethodHead, b.Filename(h), nil) if err != nil { return restic.FileInfo{}, errors.WithStack(err) } req.Header.Set("Accept", ContentTypeV2) - b.sem.GetToken() resp, err := b.client.Do(req) - b.sem.ReleaseToken() if err != nil { return restic.FileInfo{}, errors.WithStack(err) } @@ -309,19 +273,13 @@ func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, e // Remove removes the blob with the given name and type. func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { - if err := h.Valid(); err != nil { - return backoff.Permanent(err) - } - req, err := http.NewRequestWithContext(ctx, "DELETE", b.Filename(h), nil) if err != nil { return errors.WithStack(err) } req.Header.Set("Accept", ContentTypeV2) - b.sem.GetToken() resp, err := b.client.Do(req) - b.sem.ReleaseToken() if err != nil { return errors.Wrap(err, "client.Do") @@ -358,9 +316,7 @@ func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi } req.Header.Set("Accept", ContentTypeV2) - b.sem.GetToken() resp, err := b.client.Do(req) - b.sem.ReleaseToken() if err != nil { return errors.Wrap(err, "List") diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index 872fb0441..79c6453b9 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -13,12 +13,10 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" - "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" - "github.com/cenkalti/backoff/v4" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) @@ -26,7 +24,6 @@ import ( // Backend stores data on an S3 endpoint. type Backend struct { client *minio.Client - sem sema.Semaphore cfg Config layout.Layout } @@ -102,14 +99,8 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro return nil, errors.Wrap(err, "minio.New") } - sem, err := sema.New(cfg.Connections) - if err != nil { - return nil, err - } - be := &Backend{ client: client, - sem: sem, cfg: cfg, } @@ -271,15 +262,8 @@ func (be *Backend) Path() string { // Save stores data in the backend at the handle. func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { - if err := h.Valid(); err != nil { - return backoff.Permanent(err) - } - objName := be.Filename(h) - be.sem.GetToken() - defer be.sem.ReleaseToken() - opts := minio.PutObjectOptions{StorageClass: be.cfg.StorageClass} opts.ContentType = "application/octet-stream" // the only option with the high-level api is to let the library handle the checksum computation @@ -301,6 +285,9 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe // Load runs fn with a reader that yields the contents of the file at h at the // given offset. func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } @@ -321,18 +308,13 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, return nil, errors.Wrap(err, "SetRange") } - be.sem.GetToken() - ctx, cancel := context.WithCancel(ctx) - coreClient := minio.Core{Client: be.client} rd, _, _, err := coreClient.GetObject(ctx, be.cfg.Bucket, objName, opts) if err != nil { - cancel() - be.sem.ReleaseToken() return nil, err } - return be.sem.ReleaseTokenOnClose(rd, cancel), err + return rd, err } // Stat returns information about a blob. @@ -342,17 +324,14 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf opts := minio.GetObjectOptions{} - be.sem.GetToken() obj, err = be.client.GetObject(ctx, be.cfg.Bucket, objName, opts) if err != nil { - be.sem.ReleaseToken() return restic.FileInfo{}, errors.Wrap(err, "client.GetObject") } // make sure that the object is closed properly. defer func() { e := obj.Close() - be.sem.ReleaseToken() if err == nil { err = errors.Wrap(e, "Close") } @@ -370,9 +349,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { objName := be.Filename(h) - be.sem.GetToken() err := be.client.RemoveObject(ctx, be.cfg.Bucket, objName, minio.RemoveObjectOptions{}) - be.sem.ReleaseToken() if be.IsNotExist(err) { err = nil diff --git a/internal/backend/sema/backend.go b/internal/backend/sema/backend.go new file mode 100644 index 000000000..4b6f55b50 --- /dev/null +++ b/internal/backend/sema/backend.go @@ -0,0 +1,87 @@ +package sema + +import ( + "context" + "io" + + "github.com/cenkalti/backoff/v4" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" +) + +// make sure that SemaphoreBackend implements restic.Backend +var _ restic.Backend = &SemaphoreBackend{} + +// SemaphoreBackend limits the number of concurrent operations. +type SemaphoreBackend struct { + restic.Backend + sem semaphore +} + +// New creates a backend that limits the concurrent operations on the underlying backend +func New(be restic.Backend) *SemaphoreBackend { + sem, err := newSemaphore(be.Connections()) + if err != nil { + panic(err) + } + + return &SemaphoreBackend{ + Backend: be, + sem: sem, + } +} + +// Save adds new Data to the backend. +func (be *SemaphoreBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + if err := h.Valid(); err != nil { + return backoff.Permanent(err) + } + + be.sem.GetToken() + defer be.sem.ReleaseToken() + + return be.Backend.Save(ctx, h, rd) +} + +// Load runs fn with a reader that yields the contents of the file at h at the +// given offset. +func (be *SemaphoreBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + if err := h.Valid(); err != nil { + return backoff.Permanent(err) + } + if offset < 0 { + return backoff.Permanent(errors.New("offset is negative")) + } + if length < 0 { + return backoff.Permanent(errors.Errorf("invalid length %d", length)) + } + + be.sem.GetToken() + defer be.sem.ReleaseToken() + + return be.Backend.Load(ctx, h, length, offset, fn) +} + +// Stat returns information about a file in the backend. +func (be *SemaphoreBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { + if err := h.Valid(); err != nil { + return restic.FileInfo{}, backoff.Permanent(err) + } + + be.sem.GetToken() + defer be.sem.ReleaseToken() + + return be.Backend.Stat(ctx, h) +} + +// Remove deletes a file from the backend. +func (be *SemaphoreBackend) Remove(ctx context.Context, h restic.Handle) error { + if err := h.Valid(); err != nil { + return backoff.Permanent(err) + } + + be.sem.GetToken() + defer be.sem.ReleaseToken() + + return be.Backend.Remove(ctx, h) +} diff --git a/internal/backend/sema/semaphore.go b/internal/backend/sema/semaphore.go index 7ee912979..c664eef7c 100644 --- a/internal/backend/sema/semaphore.go +++ b/internal/backend/sema/semaphore.go @@ -2,64 +2,30 @@ package sema import ( - "context" - "io" - + "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" ) -// A Semaphore limits access to a restricted resource. -type Semaphore struct { +// A semaphore limits access to a restricted resource. +type semaphore struct { ch chan struct{} } -// New returns a new semaphore with capacity n. -func New(n uint) (Semaphore, error) { +// newSemaphore returns a new semaphore with capacity n. +func newSemaphore(n uint) (semaphore, error) { if n == 0 { - return Semaphore{}, errors.New("capacity must be a positive number") + return semaphore{}, errors.New("capacity must be a positive number") } - return Semaphore{ + return semaphore{ ch: make(chan struct{}, n), }, nil } // GetToken blocks until a Token is available. -func (s Semaphore) GetToken() { s.ch <- struct{}{} } +func (s semaphore) GetToken() { + s.ch <- struct{}{} + debug.Log("acquired token") +} // ReleaseToken returns a token. -func (s Semaphore) ReleaseToken() { <-s.ch } - -// ReleaseTokenOnClose wraps an io.ReadCloser to return a token on Close. -// Before returning the token, cancel, if not nil, will be run -// to free up context resources. -func (s Semaphore) ReleaseTokenOnClose(rc io.ReadCloser, cancel context.CancelFunc) io.ReadCloser { - return &wrapReader{ReadCloser: rc, sem: s, cancel: cancel} -} - -type wrapReader struct { - io.ReadCloser - eofSeen bool - sem Semaphore - cancel context.CancelFunc -} - -func (wr *wrapReader) Read(p []byte) (int, error) { - if wr.eofSeen { // XXX Why do we do this? - return 0, io.EOF - } - - n, err := wr.ReadCloser.Read(p) - if err == io.EOF { - wr.eofSeen = true - } - return n, err -} - -func (wr *wrapReader) Close() error { - err := wr.ReadCloser.Close() - if wr.cancel != nil { - wr.cancel() - } - wr.sem.ReleaseToken() - return err -} +func (s semaphore) ReleaseToken() { <-s.ch } diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index afe3fc394..e97a5f9c8 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -15,7 +15,6 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" - "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -35,7 +34,6 @@ type SFTP struct { posixRename bool - sem sema.Semaphore layout.Layout Config backend.Modes @@ -140,11 +138,7 @@ func Open(ctx context.Context, cfg Config) (*SFTP, error) { } func open(ctx context.Context, sftp *SFTP, cfg Config) (*SFTP, error) { - sem, err := sema.New(cfg.Connections) - if err != nil { - return nil, err - } - + var err error sftp.Layout, err = layout.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path) if err != nil { return nil, err @@ -158,7 +152,6 @@ func open(ctx context.Context, sftp *SFTP, cfg Config) (*SFTP, error) { sftp.Config = cfg sftp.p = cfg.Path - sftp.sem = sem sftp.Modes = m return sftp, nil } @@ -308,17 +301,10 @@ func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader return err } - if err := h.Valid(); err != nil { - return backoff.Permanent(err) - } - filename := r.Filename(h) tmpFilename := filename + "-restic-temp-" + tempSuffix() dirname := r.Dirname(h) - r.sem.GetToken() - defer r.sem.ReleaseToken() - // create new file f, err := r.c.OpenFile(tmpFilename, os.O_CREATE|os.O_EXCL|os.O_WRONLY) @@ -414,52 +400,27 @@ func (r *SFTP) Load(ctx context.Context, h restic.Handle, length int, offset int return backend.DefaultLoad(ctx, h, length, offset, r.openReader, fn) } -// wrapReader wraps an io.ReadCloser to run an additional function on Close. -type wrapReader struct { - io.ReadCloser - io.WriterTo - f func() -} - -func (wr *wrapReader) Close() error { - err := wr.ReadCloser.Close() - wr.f() - return err -} - func (r *SFTP) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - r.sem.GetToken() f, err := r.c.Open(r.Filename(h)) if err != nil { - r.sem.ReleaseToken() return nil, err } if offset > 0 { _, err = f.Seek(offset, 0) if err != nil { - r.sem.ReleaseToken() _ = f.Close() return nil, err } } - // use custom close wrapper to also provide WriteTo() on the wrapper - rd := &wrapReader{ - ReadCloser: f, - WriterTo: f, - f: func() { - r.sem.ReleaseToken() - }, - } - if length > 0 { // unlimited reads usually use io.Copy which needs WriteTo support at the underlying reader // limited reads are usually combined with io.ReadFull which reads all required bytes into a buffer in one go - return backend.LimitReadCloser(rd, int64(length)), nil + return backend.LimitReadCloser(f, int64(length)), nil } - return rd, nil + return f, nil } // Stat returns information about a blob. @@ -468,13 +429,6 @@ func (r *SFTP) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, erro return restic.FileInfo{}, err } - if err := h.Valid(); err != nil { - return restic.FileInfo{}, backoff.Permanent(err) - } - - r.sem.GetToken() - defer r.sem.ReleaseToken() - fi, err := r.c.Lstat(r.Filename(h)) if err != nil { return restic.FileInfo{}, errors.Wrap(err, "Lstat") @@ -489,9 +443,6 @@ func (r *SFTP) Remove(ctx context.Context, h restic.Handle) error { return err } - r.sem.GetToken() - defer r.sem.ReleaseToken() - return r.c.Remove(r.Filename(h)) } @@ -501,9 +452,7 @@ func (r *SFTP) List(ctx context.Context, t restic.FileType, fn func(restic.FileI basedir, subdirs := r.Basedir(t) walker := r.c.Walk(basedir) for { - r.sem.GetToken() ok := walker.Step() - r.sem.ReleaseToken() if !ok { break } diff --git a/internal/backend/swift/swift.go b/internal/backend/swift/swift.go index 99940df5c..dbf4ba0d1 100644 --- a/internal/backend/swift/swift.go +++ b/internal/backend/swift/swift.go @@ -15,12 +15,10 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" - "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" - "github.com/cenkalti/backoff/v4" "github.com/ncw/swift/v2" ) @@ -28,7 +26,6 @@ import ( type beSwift struct { conn *swift.Connection connections uint - sem sema.Semaphore container string // Container name prefix string // Prefix of object names in the container layout.Layout @@ -42,11 +39,6 @@ var _ restic.Backend = &beSwift{} func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { debug.Log("config %#v", cfg) - sem, err := sema.New(cfg.Connections) - if err != nil { - return nil, err - } - be := &beSwift{ conn: &swift.Connection{ UserName: cfg.UserName, @@ -72,7 +64,6 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend Transport: rt, }, connections: cfg.Connections, - sem: sem, container: cfg.Container, prefix: cfg.Prefix, Layout: &layout.DefaultLayout{ @@ -159,27 +150,17 @@ func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int, debug.Log("Load(%v) send range %v", h, headers["Range"]) } - be.sem.GetToken() obj, _, err := be.conn.ObjectOpen(ctx, be.container, objName, false, headers) if err != nil { - be.sem.ReleaseToken() return nil, errors.Wrap(err, "conn.ObjectOpen") } - return be.sem.ReleaseTokenOnClose(obj, nil), nil + return obj, nil } // Save stores data in the backend at the handle. func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { - if err := h.Valid(); err != nil { - return backoff.Permanent(err) - } - objName := be.Filename(h) - - be.sem.GetToken() - defer be.sem.ReleaseToken() - encoding := "binary/octet-stream" debug.Log("PutObject(%v, %v, %v)", be.container, objName, encoding) @@ -196,9 +177,6 @@ func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe func (be *beSwift) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { objName := be.Filename(h) - be.sem.GetToken() - defer be.sem.ReleaseToken() - obj, _, err := be.conn.Object(ctx, be.container, objName) if err != nil { return restic.FileInfo{}, errors.Wrap(err, "conn.Object") @@ -211,9 +189,6 @@ func (be *beSwift) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error { objName := be.Filename(h) - be.sem.GetToken() - defer be.sem.ReleaseToken() - err := be.conn.ObjectDelete(ctx, be.container, objName) return errors.Wrap(err, "conn.ObjectDelete") } @@ -226,9 +201,7 @@ func (be *beSwift) List(ctx context.Context, t restic.FileType, fn func(restic.F err := be.conn.ObjectsWalk(ctx, be.container, &swift.ObjectsOpts{Prefix: prefix}, func(ctx context.Context, opts *swift.ObjectsOpts) (interface{}, error) { - be.sem.GetToken() newObjects, err := be.conn.Objects(ctx, be.container, opts) - be.sem.ReleaseToken() if err != nil { return nil, errors.Wrap(err, "conn.ObjectNames") diff --git a/internal/backend/utils.go b/internal/backend/utils.go index 1c1607e04..bf8a7ad6d 100644 --- a/internal/backend/utils.go +++ b/internal/backend/utils.go @@ -6,7 +6,6 @@ import ( "fmt" "io" - "github.com/cenkalti/backoff/v4" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -63,15 +62,7 @@ func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser { func DefaultLoad(ctx context.Context, h restic.Handle, length int, offset int64, openReader func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error), fn func(rd io.Reader) error) error { - if err := h.Valid(); err != nil { - return backoff.Permanent(err) - } - if offset < 0 { - return errors.New("offset is negative") - } - if length < 0 { - return errors.Errorf("invalid length %d", length) - } + rd, err := openReader(ctx, h, length, offset) if err != nil { return err From 803640ba4b0befb02aa4aca3700344ce10add88f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 7 Apr 2023 23:22:00 +0200 Subject: [PATCH 058/127] backend: remove a few unnecessary debug logs --- internal/backend/mem/mem_backend.go | 2 -- internal/backend/rest/rest.go | 1 - internal/backend/s3/s3.go | 3 --- internal/backend/swift/swift.go | 5 ----- 4 files changed, 11 deletions(-) diff --git a/internal/backend/mem/mem_backend.go b/internal/backend/mem/mem_backend.go index 4db4c9821..618ef5752 100644 --- a/internal/backend/mem/mem_backend.go +++ b/internal/backend/mem/mem_backend.go @@ -144,8 +144,6 @@ func (be *MemoryBackend) Remove(ctx context.Context, h restic.Handle) error { be.m.Lock() defer be.m.Unlock() - debug.Log("Remove %v", h) - h.ContainedBlobType = restic.InvalidBlob if _, ok := be.data[h]; !ok { return errNotFound diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index a88e26daa..f9030d076 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -207,7 +207,6 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o } req.Header.Set("Range", byteRange) req.Header.Set("Accept", ContentTypeV2) - debug.Log("Load(%v) send range %v", h, byteRange) resp, err := b.client.Do(req) diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index 79c6453b9..591ad2185 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -271,7 +271,6 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe // only use multipart uploads for very large files opts.PartSize = 200 * 1024 * 1024 - debug.Log("PutObject(%v, %v, %v)", be.cfg.Bucket, objName, rd.Length()) info, err := be.client.PutObject(ctx, be.cfg.Bucket, objName, io.NopCloser(rd), int64(rd.Length()), opts) // sanity check @@ -297,10 +296,8 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, var err error if length > 0 { - debug.Log("range: %v-%v", offset, offset+int64(length)-1) err = opts.SetRange(offset, offset+int64(length)-1) } else if offset > 0 { - debug.Log("range: %v-", offset) err = opts.SetRange(offset, 0) } diff --git a/internal/backend/swift/swift.go b/internal/backend/swift/swift.go index dbf4ba0d1..fcdbe6634 100644 --- a/internal/backend/swift/swift.go +++ b/internal/backend/swift/swift.go @@ -146,10 +146,6 @@ func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int, headers["Range"] = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1) } - if _, ok := headers["Range"]; ok { - debug.Log("Load(%v) send range %v", h, headers["Range"]) - } - obj, _, err := be.conn.ObjectOpen(ctx, be.container, objName, false, headers) if err != nil { return nil, errors.Wrap(err, "conn.ObjectOpen") @@ -163,7 +159,6 @@ func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe objName := be.Filename(h) encoding := "binary/octet-stream" - debug.Log("PutObject(%v, %v, %v)", be.container, objName, encoding) hdr := swift.Headers{"Content-Length": strconv.FormatInt(rd.Length(), 10)} _, err := be.conn.ObjectPut(ctx, be.container, objName, rd, true, hex.EncodeToString(rd.Hash()), From 45244fdf683d269edb1100d28afd2f817ff5d914 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 7 Apr 2023 23:24:14 +0200 Subject: [PATCH 059/127] backend: remove parameter validation tests These parameter validations have been factored out into SemaphoreBackend. --- internal/backend/dryrun/dry_backend_test.go | 2 -- internal/backend/test/tests.go | 17 +---------------- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/internal/backend/dryrun/dry_backend_test.go b/internal/backend/dryrun/dry_backend_test.go index 6b8f74e0f..69716c340 100644 --- a/internal/backend/dryrun/dry_backend_test.go +++ b/internal/backend/dryrun/dry_backend_test.go @@ -40,11 +40,9 @@ func TestDry(t *testing.T) { {d, "delete", "", "", ""}, {d, "stat", "a", "", "not found"}, {d, "list", "", "", ""}, - {d, "save", "", "", "invalid"}, {m, "save", "a", "baz", ""}, // save a directly to the mem backend {d, "save", "b", "foob", ""}, // b is not saved {d, "save", "b", "xxx", ""}, // no error as b is not saved - {d, "stat", "", "", "invalid"}, {d, "stat", "a", "a 3", ""}, {d, "load", "a", "baz", ""}, {d, "load", "b", "", "not found"}, diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go index b98af59c3..53a10f446 100644 --- a/internal/backend/test/tests.go +++ b/internal/backend/test/tests.go @@ -124,17 +124,7 @@ func (s *Suite) TestLoad(t *testing.T) { b := s.open(t) defer s.close(t, b) - noop := func(rd io.Reader) error { - return nil - } - - err := b.Load(context.TODO(), restic.Handle{}, 0, 0, noop) - if err == nil { - t.Fatalf("Load() did not return an error for invalid handle") - } - test.Assert(t, !b.IsNotExist(err), "IsNotExist() should not accept an invalid handle error: %v", err) - - err = testLoad(b, restic.Handle{Type: restic.PackFile, Name: "foobar"}, 0, 0) + err := testLoad(b, restic.Handle{Type: restic.PackFile, Name: "foobar"}, 0, 0) if err == nil { t.Fatalf("Load() did not return an error for non-existing blob") } @@ -153,11 +143,6 @@ func (s *Suite) TestLoad(t *testing.T) { t.Logf("saved %d bytes as %v", length, handle) - err = b.Load(context.TODO(), handle, 100, -1, noop) - if err == nil { - t.Fatalf("Load() returned no error for negative offset!") - } - err = b.Load(context.TODO(), handle, 0, 0, func(rd io.Reader) error { _, err := io.Copy(io.Discard, rd) if err != nil { From 05abc6d6f5766d73c079a596db0d04ab8e777942 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 7 Apr 2023 23:56:16 +0200 Subject: [PATCH 060/127] backend: deduplicate implementation of Delete() method --- internal/backend/azure/azure.go | 23 +---------------------- internal/backend/b2/b2.go | 27 +-------------------------- internal/backend/gs/gs.go | 23 +---------------------- internal/backend/rest/rest.go | 28 ++-------------------------- internal/backend/s3/s3.go | 23 +---------------------- internal/backend/swift/swift.go | 28 +--------------------------- internal/backend/utils.go | 25 +++++++++++++++++++++++++ 7 files changed, 32 insertions(+), 145 deletions(-) diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index 82d55960f..9a3695f0f 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -384,30 +384,9 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F return ctx.Err() } -// Remove keys for a specified backend type. -func (be *Backend) removeKeys(ctx context.Context, t restic.FileType) error { - return be.List(ctx, t, func(fi restic.FileInfo) error { - return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) - }) -} - // Delete removes all restic keys in the bucket. It will not remove the bucket itself. func (be *Backend) Delete(ctx context.Context) error { - alltypes := []restic.FileType{ - restic.PackFile, - restic.KeyFile, - restic.LockFile, - restic.SnapshotFile, - restic.IndexFile} - - for _, t := range alltypes { - err := be.removeKeys(ctx, t) - if err != nil { - return nil - } - } - - return be.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) + return backend.DefaultDelete(ctx, be) } // Close does nothing diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go index 0827f727b..738df198d 100644 --- a/internal/backend/b2/b2.go +++ b/internal/backend/b2/b2.go @@ -312,34 +312,9 @@ func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic return nil } -// Remove keys for a specified backend type. -func (be *b2Backend) removeKeys(ctx context.Context, t restic.FileType) error { - return be.List(ctx, t, func(fi restic.FileInfo) error { - return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) - }) -} - // Delete removes all restic keys in the bucket. It will not remove the bucket itself. func (be *b2Backend) Delete(ctx context.Context) error { - alltypes := []restic.FileType{ - restic.PackFile, - restic.KeyFile, - restic.LockFile, - restic.SnapshotFile, - restic.IndexFile} - - for _, t := range alltypes { - err := be.removeKeys(ctx, t) - if err != nil { - return nil - } - } - err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) - if err != nil && be.IsNotExist(err) { - err = nil - } - - return err + return backend.DefaultDelete(ctx, be) } // Close does nothing diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index 12458a79c..de798ac92 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -339,30 +339,9 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F return ctx.Err() } -// Remove keys for a specified backend type. -func (be *Backend) removeKeys(ctx context.Context, t restic.FileType) error { - return be.List(ctx, t, func(fi restic.FileInfo) error { - return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) - }) -} - // Delete removes all restic keys in the bucket. It will not remove the bucket itself. func (be *Backend) Delete(ctx context.Context) error { - alltypes := []restic.FileType{ - restic.PackFile, - restic.KeyFile, - restic.LockFile, - restic.SnapshotFile, - restic.IndexFile} - - for _, t := range alltypes { - err := be.removeKeys(ctx, t) - if err != nil { - return nil - } - } - - return be.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) + return backend.DefaultDelete(ctx, be) } // Close does nothing. diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index f9030d076..7be5a07c7 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -11,6 +11,7 @@ import ( "path" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" @@ -411,32 +412,7 @@ func (b *Backend) Close() error { return nil } -// Remove keys for a specified backend type. -func (b *Backend) removeKeys(ctx context.Context, t restic.FileType) error { - return b.List(ctx, t, func(fi restic.FileInfo) error { - return b.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) - }) -} - // Delete removes all data in the backend. func (b *Backend) Delete(ctx context.Context) error { - alltypes := []restic.FileType{ - restic.PackFile, - restic.KeyFile, - restic.LockFile, - restic.SnapshotFile, - restic.IndexFile} - - for _, t := range alltypes { - err := b.removeKeys(ctx, t) - if err != nil { - return nil - } - } - - err := b.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) - if err != nil && b.IsNotExist(err) { - return nil - } - return err + return backend.DefaultDelete(ctx, b) } diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index 591ad2185..7b7a761ce 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -411,30 +411,9 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F return ctx.Err() } -// Remove keys for a specified backend type. -func (be *Backend) removeKeys(ctx context.Context, t restic.FileType) error { - return be.List(ctx, restic.PackFile, func(fi restic.FileInfo) error { - return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) - }) -} - // Delete removes all restic keys in the bucket. It will not remove the bucket itself. func (be *Backend) Delete(ctx context.Context) error { - alltypes := []restic.FileType{ - restic.PackFile, - restic.KeyFile, - restic.LockFile, - restic.SnapshotFile, - restic.IndexFile} - - for _, t := range alltypes { - err := be.removeKeys(ctx, t) - if err != nil { - return nil - } - } - - return be.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) + return backend.DefaultDelete(ctx, be) } // Close does nothing diff --git a/internal/backend/swift/swift.go b/internal/backend/swift/swift.go index fcdbe6634..cfa9ed665 100644 --- a/internal/backend/swift/swift.go +++ b/internal/backend/swift/swift.go @@ -231,13 +231,6 @@ func (be *beSwift) List(ctx context.Context, t restic.FileType, fn func(restic.F return ctx.Err() } -// Remove keys for a specified backend type. -func (be *beSwift) removeKeys(ctx context.Context, t restic.FileType) error { - return be.List(ctx, t, func(fi restic.FileInfo) error { - return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) - }) -} - // IsNotExist returns true if the error is caused by a not existing file. func (be *beSwift) IsNotExist(err error) bool { var e *swift.Error @@ -247,26 +240,7 @@ func (be *beSwift) IsNotExist(err error) bool { // Delete removes all restic objects in the container. // It will not remove the container itself. func (be *beSwift) Delete(ctx context.Context) error { - alltypes := []restic.FileType{ - restic.PackFile, - restic.KeyFile, - restic.LockFile, - restic.SnapshotFile, - restic.IndexFile} - - for _, t := range alltypes { - err := be.removeKeys(ctx, t) - if err != nil { - return nil - } - } - - err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) - if err != nil && !be.IsNotExist(err) { - return err - } - - return nil + return backend.DefaultDelete(ctx, be) } // Close does nothing diff --git a/internal/backend/utils.go b/internal/backend/utils.go index bf8a7ad6d..cd6614f34 100644 --- a/internal/backend/utils.go +++ b/internal/backend/utils.go @@ -75,6 +75,31 @@ func DefaultLoad(ctx context.Context, h restic.Handle, length int, offset int64, return rd.Close() } +// DefaultDelete removes all restic keys in the bucket. It will not remove the bucket itself. +func DefaultDelete(ctx context.Context, be restic.Backend) error { + alltypes := []restic.FileType{ + restic.PackFile, + restic.KeyFile, + restic.LockFile, + restic.SnapshotFile, + restic.IndexFile} + + for _, t := range alltypes { + err := be.List(ctx, t, func(fi restic.FileInfo) error { + return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) + }) + if err != nil { + return nil + } + } + err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) + if err != nil && be.IsNotExist(err) { + err = nil + } + + return err +} + type memorizedLister struct { fileInfos []restic.FileInfo tpe restic.FileType From 616926d2c19387889bbf00f765f5c131a521545b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 7 Apr 2023 23:16:08 +0200 Subject: [PATCH 061/127] gs: use IsNotExist to check error --- internal/backend/gs/gs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index de798ac92..f310fbef5 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -286,7 +286,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { err := be.bucket.Object(objName).Delete(ctx) - if err == storage.ErrObjectNotExist { + if be.IsNotExist(err) { err = nil } From c934c99d4124d4033b7cbd99b79fc8ebe377d804 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 8 Apr 2023 11:59:44 +0200 Subject: [PATCH 062/127] gs: replace usage of context.Background() --- cmd/restic/global.go | 2 +- internal/backend/gs/gs.go | 3 +-- internal/backend/gs/gs_test.go | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 8d34f8ddb..537fe02fe 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -801,7 +801,7 @@ func create(ctx context.Context, s string, opts options.Options) (restic.Backend case "s3": be, err = s3.Create(ctx, cfg.(s3.Config), rt) case "gs": - be, err = gs.Create(cfg.(gs.Config), rt) + be, err = gs.Create(ctx, cfg.(gs.Config), rt) case "azure": be, err = azure.Create(ctx, cfg.(azure.Config), rt) case "swift": diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index f310fbef5..62e5c4954 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -124,14 +124,13 @@ func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) { // // The service account must have the "storage.buckets.create" permission to // create a bucket the does not yet exist. -func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { be, err := open(cfg, rt) if err != nil { return nil, errors.Wrap(err, "open") } // Try to determine if the bucket exists. If it does not, try to create it. - ctx := context.Background() exists, err := be.bucketExists(ctx, be.bucket) if err != nil { if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusForbidden { diff --git a/internal/backend/gs/gs_test.go b/internal/backend/gs/gs_test.go index 77f8986f1..19ae8b829 100644 --- a/internal/backend/gs/gs_test.go +++ b/internal/backend/gs/gs_test.go @@ -42,7 +42,7 @@ func newGSTestSuite(t testing.TB) *test.Suite { Create: func(config interface{}) (restic.Backend, error) { cfg := config.(gs.Config) - be, err := gs.Create(cfg, tr) + be, err := gs.Create(context.Background(), cfg, tr) if err != nil { return nil, err } From 6042df075fba7d8113586181cd0ab332b5304aaa Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 8 Apr 2023 12:53:43 +0200 Subject: [PATCH 063/127] migrations: Fix S3 backend detection --- internal/backend/limiter/limiter_backend.go | 2 ++ internal/backend/logger/log.go | 2 ++ internal/backend/retry/backend_retry.go | 4 +++ internal/backend/sema/backend.go | 4 +++ internal/cache/backend.go | 4 +++ internal/migrations/s3_layout.go | 29 +++++++++++---------- internal/migrations/s3_layout_test.go | 27 +++++++++++++++++++ internal/restic/backend.go | 5 ++++ 8 files changed, 63 insertions(+), 14 deletions(-) create mode 100644 internal/migrations/s3_layout_test.go diff --git a/internal/backend/limiter/limiter_backend.go b/internal/backend/limiter/limiter_backend.go index f1b508327..7fcca59cc 100644 --- a/internal/backend/limiter/limiter_backend.go +++ b/internal/backend/limiter/limiter_backend.go @@ -46,6 +46,8 @@ func (r rateLimitedBackend) Load(ctx context.Context, h restic.Handle, length in }) } +func (r rateLimitedBackend) Unwrap() restic.Backend { return r.Backend } + type limitedReader struct { io.Reader writerTo io.WriterTo diff --git a/internal/backend/logger/log.go b/internal/backend/logger/log.go index 4623d8021..6c860cfae 100644 --- a/internal/backend/logger/log.go +++ b/internal/backend/logger/log.go @@ -75,3 +75,5 @@ func (be *Backend) Close() error { debug.Log(" close err %v", err) return err } + +func (be *Backend) Unwrap() restic.Backend { return be.Backend } diff --git a/internal/backend/retry/backend_retry.go b/internal/backend/retry/backend_retry.go index b5f2706f4..9c51efedc 100644 --- a/internal/backend/retry/backend_retry.go +++ b/internal/backend/retry/backend_retry.go @@ -191,3 +191,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F return err } + +func (be *Backend) Unwrap() restic.Backend { + return be.Backend +} diff --git a/internal/backend/sema/backend.go b/internal/backend/sema/backend.go index 4b6f55b50..9f6dfbadd 100644 --- a/internal/backend/sema/backend.go +++ b/internal/backend/sema/backend.go @@ -85,3 +85,7 @@ func (be *SemaphoreBackend) Remove(ctx context.Context, h restic.Handle) error { return be.Backend.Remove(ctx, h) } + +func (be *SemaphoreBackend) Unwrap() restic.Backend { + return be.Backend +} diff --git a/internal/cache/backend.go b/internal/cache/backend.go index a707f8243..08ec1facd 100644 --- a/internal/cache/backend.go +++ b/internal/cache/backend.go @@ -211,3 +211,7 @@ func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, e func (b *Backend) IsNotExist(err error) bool { return b.Backend.IsNotExist(err) } + +func (b *Backend) Unwrap() restic.Backend { + return b.Backend +} diff --git a/internal/migrations/s3_layout.go b/internal/migrations/s3_layout.go index d42b94bf8..a5293ef16 100644 --- a/internal/migrations/s3_layout.go +++ b/internal/migrations/s3_layout.go @@ -8,7 +8,6 @@ import ( "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/s3" - "github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -22,24 +21,26 @@ func init() { // "default" layout. type S3Layout struct{} -func toS3Backend(repo restic.Repository) *s3.Backend { - b := repo.Backend() - // unwrap cache - if be, ok := b.(*cache.Backend); ok { - b = be.Backend - } +func toS3Backend(b restic.Backend) *s3.Backend { + for b != nil { + if be, ok := b.(*s3.Backend); ok { + return be + } - be, ok := b.(*s3.Backend) - if !ok { - debug.Log("backend is not s3") - return nil + if be, ok := b.(restic.BackendUnwrapper); ok { + b = be.Unwrap() + } else { + // not the backend we're looking for + break + } } - return be + debug.Log("backend is not s3") + return nil } // Check tests whether the migration can be applied. func (m *S3Layout) Check(ctx context.Context, repo restic.Repository) (bool, string, error) { - be := toS3Backend(repo) + be := toS3Backend(repo.Backend()) if be == nil { debug.Log("backend is not s3") return false, "backend is not s3", nil @@ -91,7 +92,7 @@ func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l layout.Layou // Apply runs the migration. func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error { - be := toS3Backend(repo) + be := toS3Backend(repo.Backend()) if be == nil { debug.Log("backend is not s3") return errors.New("backend is not s3") diff --git a/internal/migrations/s3_layout_test.go b/internal/migrations/s3_layout_test.go new file mode 100644 index 000000000..ad0eedea6 --- /dev/null +++ b/internal/migrations/s3_layout_test.go @@ -0,0 +1,27 @@ +package migrations + +import ( + "testing" + + "github.com/restic/restic/internal/backend/mock" + "github.com/restic/restic/internal/backend/s3" + "github.com/restic/restic/internal/cache" + "github.com/restic/restic/internal/test" +) + +func TestS3UnwrapBackend(t *testing.T) { + // toS3Backend(b restic.Backend) *s3.Backend + + m := mock.NewBackend() + test.Assert(t, toS3Backend(m) == nil, "mock backend is not an s3 backend") + + // uninitialized fake backend for testing + s3 := &s3.Backend{} + test.Assert(t, toS3Backend(s3) == s3, "s3 was not returned") + + c := &cache.Backend{Backend: s3} + test.Assert(t, toS3Backend(c) == s3, "failed to unwrap s3 backend") + + c.Backend = m + test.Assert(t, toS3Backend(c) == nil, "a wrapped mock backend is not an s3 backend") +} diff --git a/internal/restic/backend.go b/internal/restic/backend.go index bc139fc8b..b01071132 100644 --- a/internal/restic/backend.go +++ b/internal/restic/backend.go @@ -70,6 +70,11 @@ type Backend interface { Delete(ctx context.Context) error } +type BackendUnwrapper interface { + // Unwrap returns the underlying backend or nil if there is none. + Unwrap() Backend +} + // FileInfo is contains information about a file in the backend. type FileInfo struct { Size int64 From 48e065d9710f07234ffa1ff93d0065a895f58531 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 14 Apr 2023 20:56:46 +0200 Subject: [PATCH 064/127] Sync dependency upgrades from master github.com/Azure/azure-sdk-for-go/sdk/storage/azblob still uses v0.5.1 as upgrading it would increase the minimum Go version on Solaris to 1.20. --- changelog/unreleased/issue-4275 | 4 ++ go.mod | 44 ++++++++-------- go.sum | 92 ++++++++++++++++----------------- 3 files changed, 72 insertions(+), 68 deletions(-) create mode 100644 changelog/unreleased/issue-4275 diff --git a/changelog/unreleased/issue-4275 b/changelog/unreleased/issue-4275 new file mode 100644 index 000000000..3fbe093fb --- /dev/null +++ b/changelog/unreleased/issue-4275 @@ -0,0 +1,4 @@ +Security: Update golang.org/x/net to address CVE-2022-41723 + +https://github.com/restic/restic/issues/4275 +https://github.com/restic/restic/pull/4213 diff --git a/go.mod b/go.mod index a172c6992..d0e514220 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module github.com/restic/restic require ( - cloud.google.com/go/storage v1.29.0 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 + cloud.google.com/go/storage v1.30.1 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 github.com/anacrolix/fuse v0.2.0 github.com/cenkalti/backoff/v4 v4.2.0 @@ -12,9 +12,9 @@ require ( github.com/google/go-cmp v0.5.9 github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/juju/ratelimit v1.0.2 - github.com/klauspost/compress v1.15.15 + github.com/klauspost/compress v1.16.0 github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5 - github.com/minio/minio-go/v7 v7.0.47 + github.com/minio/minio-go/v7 v7.0.50 github.com/minio/sha256-simd v1.0.0 github.com/ncw/swift/v2 v2.0.1 github.com/pkg/errors v0.9.1 @@ -24,35 +24,35 @@ require ( github.com/restic/chunker v0.4.0 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 - golang.org/x/crypto v0.5.0 - golang.org/x/net v0.5.0 - golang.org/x/oauth2 v0.4.0 + golang.org/x/crypto v0.7.0 + golang.org/x/net v0.8.0 + golang.org/x/oauth2 v0.6.0 golang.org/x/sync v0.1.0 - golang.org/x/sys v0.4.0 - golang.org/x/term v0.4.0 - golang.org/x/text v0.6.0 - google.golang.org/api v0.108.0 + golang.org/x/sys v0.6.0 + golang.org/x/term v0.6.0 + golang.org/x/text v0.8.0 + google.golang.org/api v0.116.0 ) require ( - cloud.google.com/go v0.108.0 // indirect - cloud.google.com/go/compute v1.15.1 // indirect + cloud.google.com/go v0.110.0 // indirect + cloud.google.com/go/compute v1.19.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.10.0 // indirect + cloud.google.com/go/iam v0.13.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/dnaeon/go-vcr v1.2.0 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/fgprof v0.9.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b // indirect github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect - github.com/googleapis/gax-go/v2 v2.7.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect + github.com/googleapis/gax-go/v2 v2.8.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.3 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/kr/fs v0.1.0 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -63,9 +63,9 @@ require ( go.opencensus.io v0.24.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect - google.golang.org/grpc v1.52.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 // indirect + google.golang.org/grpc v1.54.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 08069a411..ce664c15a 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,17 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.108.0 h1:xntQwnfn8oHGX0crLVinvHM+AhXvi3QHQIEcX/2hiWk= -cloud.google.com/go v0.108.0/go.mod h1:lNUfQqusBJp0bgAg6qrHgYFYbTB+dOiob1itwnlD33Q= -cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE= -cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= +cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go/compute v1.19.0 h1:+9zda3WGgW1ZSTlVppLCYFIr48Pa35q1uG2N1itbCEQ= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/iam v0.10.0 h1:fpP/gByFs6US1ma53v7VxhvbJpO2Aapng6wabJ99MuI= -cloud.google.com/go/iam v0.10.0/go.mod h1:nXAECrMt2qHpF6RZUZseteD6QyanL68reN4OXPw0UWM= -cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= -cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= -cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= +cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= +cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJVR2cMC8lvZcimipiEY= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= @@ -39,8 +39,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/elithrar/simple-scrypt v1.3.0 h1:KIlOlxdoQf9JWKl5lMAJ28SY2URB0XTRDn2TckyzAZg= @@ -70,8 +70,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -82,17 +82,17 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b h1:8htHrh2bw9c7Idkb7YNac+ZpTqLMjRpI+FWu51ltaQc= github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.1 h1:RY7tHKZcRlk788d5WSo/e83gOyyy742E8GSs771ySpg= -github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= -github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= -github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/gax-go/v2 v2.8.0 h1:UBtEZqx1bjXtOQ5BVTkuYghXrr3N4V123VKJK67vJZc= +github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= @@ -103,12 +103,12 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI= github.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= -github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= -github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= +github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= +github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= -github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5 h1:OUlGa6AAolmjyPtILbMJ8vHayz5wd4wBUloheGcMhfA= @@ -116,8 +116,8 @@ github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5/go.mod h1:4FCXMUWo9 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.47 h1:sLiuCKGSIcn/MI6lREmTzX91DX/oRau4ia0j6e6eOSs= -github.com/minio/minio-go/v7 v7.0.47/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw= +github.com/minio/minio-go/v7 v7.0.50 h1:4IL4V8m/kI90ZL6GupCARZVrBv8/XrcKcJhaJ3iz68k= +github.com/minio/minio-go/v7 v7.0.50/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -172,8 +172,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -189,11 +189,11 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= -golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -214,17 +214,17 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= -golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -237,8 +237,8 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/api v0.108.0 h1:WVBc/faN0DkKtR43Q/7+tPny9ZoLZdIiAyG5Q9vFClg= -google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.116.0 h1:09tOPVufPwfm5W4aA8EizGHJ7BcoRDsIareM2a15gO4= +google.golang.org/api v0.116.0/go.mod h1:9cD4/t6uvd9naoEJFA+M96d0IuB6BqFuyhpw68+mRGg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= @@ -246,15 +246,15 @@ google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 h1:0BOZf6qNozI3pkN3fJLwNubheHJYHhMh91GRFOWWK08= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.52.0 h1:kd48UiU7EHsV4rnLyOJRuP/Il/UHE7gdDAQ+SZI7nZk= -google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= +google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -266,8 +266,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= From 07a44a88f2ebc9456ef2a75d2aa2a4ff25ccdc5d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 14 Apr 2023 21:53:55 +0200 Subject: [PATCH 065/127] Fix snapshot filtering for relative paths in the backup command The snapshot filtering internally converts relative paths to absolute ones to ensure that the parent snapshots selection works for backups of relative paths. --- internal/restic/snapshot_find.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/restic/snapshot_find.go b/internal/restic/snapshot_find.go index 4d4bb4957..8d6f8c4b1 100644 --- a/internal/restic/snapshot_find.go +++ b/internal/restic/snapshot_find.go @@ -46,6 +46,7 @@ func (f *SnapshotFilter) findLatest(ctx context.Context, be Lister, loader Loade } absTargets = append(absTargets, filepath.Clean(target)) } + f.Paths = absTargets var latest *Snapshot From 09cddb89279d426d1807b78ee93b366d126502c9 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 17 Apr 2023 21:00:45 +0200 Subject: [PATCH 066/127] rewrite: log snapshot saved before removal of the old snapshot The snapshot was already saved before removing the old snapshot. Only the log messages were printed in the wrong order. --- cmd/restic/cmd_rewrite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index 8a1b860ed..744686390 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -136,6 +136,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti if err != nil { return false, err } + Verbosef("saved new snapshot %v\n", id.Str()) if opts.Forget { h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} @@ -145,7 +146,6 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti debug.Log("removed old snapshot %v", sn.ID()) Verbosef("removed old snapshot %v\n", sn.ID().Str()) } - Verbosef("saved new snapshot %v\n", id.Str()) return true, nil } From 3001dd8c2b759037779edad6b25fc2117ec62c3a Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 21 Apr 2023 22:35:02 +0200 Subject: [PATCH 067/127] Add test to verify that the backup parent is correctly selected --- cmd/restic/integration_test.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index dbefe83a7..10ebbaf13 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -385,6 +385,33 @@ func TestBackupWithRelativePath(t *testing.T) { rtest.Assert(t, latestSn.Parent != nil && latestSn.Parent.Equal(firstSnapshotID), "second snapshot selected unexpected parent %v instead of %v", latestSn.Parent, firstSnapshotID) } +func TestBackupParentSelection(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{} + + // first backup + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata/0/0"}, opts, env.gopts) + snapshotIDs := testRunList(t, "snapshots", env.gopts) + rtest.Assert(t, len(snapshotIDs) == 1, "expected one snapshot, got %v", snapshotIDs) + firstSnapshotID := snapshotIDs[0] + + // second backup, sibling path + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata/0/tests"}, opts, env.gopts) + snapshotIDs = testRunList(t, "snapshots", env.gopts) + rtest.Assert(t, len(snapshotIDs) == 2, "expected two snapshots, got %v", snapshotIDs) + + // third backup, incremental for the first backup + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata/0/0"}, opts, env.gopts) + + // test that the correct parent snapshot was used + latestSn, _ := testRunSnapshots(t, env.gopts) + rtest.Assert(t, latestSn != nil, "missing latest snapshot") + rtest.Assert(t, latestSn.Parent != nil && latestSn.Parent.Equal(firstSnapshotID), "third snapshot selected unexpected parent %v instead of %v", latestSn.Parent, firstSnapshotID) +} + func TestDryRunBackup(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() From f27750e27004108d64fc6e313e7824b59823d0d5 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 21 Apr 2023 22:44:45 +0200 Subject: [PATCH 068/127] backend/sema: rename type to connectionLimitedBackend --- internal/backend/sema/backend.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/backend/sema/backend.go b/internal/backend/sema/backend.go index 9f6dfbadd..2294ef0b2 100644 --- a/internal/backend/sema/backend.go +++ b/internal/backend/sema/backend.go @@ -9,30 +9,30 @@ import ( "github.com/restic/restic/internal/restic" ) -// make sure that SemaphoreBackend implements restic.Backend -var _ restic.Backend = &SemaphoreBackend{} +// make sure that connectionLimitedBackend implements restic.Backend +var _ restic.Backend = &connectionLimitedBackend{} -// SemaphoreBackend limits the number of concurrent operations. -type SemaphoreBackend struct { +// connectionLimitedBackend limits the number of concurrent operations. +type connectionLimitedBackend struct { restic.Backend sem semaphore } // New creates a backend that limits the concurrent operations on the underlying backend -func New(be restic.Backend) *SemaphoreBackend { +func New(be restic.Backend) restic.Backend { sem, err := newSemaphore(be.Connections()) if err != nil { panic(err) } - return &SemaphoreBackend{ + return &connectionLimitedBackend{ Backend: be, sem: sem, } } // Save adds new Data to the backend. -func (be *SemaphoreBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *connectionLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { if err := h.Valid(); err != nil { return backoff.Permanent(err) } @@ -45,7 +45,7 @@ func (be *SemaphoreBackend) Save(ctx context.Context, h restic.Handle, rd restic // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *SemaphoreBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *connectionLimitedBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { if err := h.Valid(); err != nil { return backoff.Permanent(err) } @@ -63,7 +63,7 @@ func (be *SemaphoreBackend) Load(ctx context.Context, h restic.Handle, length in } // Stat returns information about a file in the backend. -func (be *SemaphoreBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (be *connectionLimitedBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { if err := h.Valid(); err != nil { return restic.FileInfo{}, backoff.Permanent(err) } @@ -75,7 +75,7 @@ func (be *SemaphoreBackend) Stat(ctx context.Context, h restic.Handle) (restic.F } // Remove deletes a file from the backend. -func (be *SemaphoreBackend) Remove(ctx context.Context, h restic.Handle) error { +func (be *connectionLimitedBackend) Remove(ctx context.Context, h restic.Handle) error { if err := h.Valid(); err != nil { return backoff.Permanent(err) } @@ -86,6 +86,6 @@ func (be *SemaphoreBackend) Remove(ctx context.Context, h restic.Handle) error { return be.Backend.Remove(ctx, h) } -func (be *SemaphoreBackend) Unwrap() restic.Backend { +func (be *connectionLimitedBackend) Unwrap() restic.Backend { return be.Backend } From affd04c125c69d1cd2a9702c3e024981ddcbe752 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 21 Apr 2023 22:56:31 +0200 Subject: [PATCH 069/127] doc: recommend `cat config` to check if a repository exists This will be much faster if a large number of snapshots exists. --- doc/075_scripting.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index 712a70244..dc7f782dc 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -22,18 +22,18 @@ Check if a repository is already initialized You may find a need to check if a repository is already initialized, perhaps to prevent your script from initializing a repository multiple -times. The command ``snapshots`` may be used for this purpose: +times. The command ``cat config`` may be used for this purpose: .. code-block:: console - $ restic -r /srv/restic-repo snapshots - Fatal: unable to open config file: Stat: stat /srv/restic-repo/config: no such file or directory + $ restic -r /srv/restic-repo cat config + Fatal: unable to open config file: stat /srv/restic-repo/config: no such file or directory Is there a repository at the following location? /srv/restic-repo If a repository does not exist, restic will return a non-zero exit code and print an error message. Note that restic will also return a non-zero exit code if a different error is encountered (e.g.: incorrect password -to ``snapshots``) and it may print a different error message. If there -are no errors, restic will return a zero exit code and print all the -snapshots. +to ``cat config``) and it may print a different error message. If there +are no errors, restic will return a zero exit code and print the repository +metadata. From 756f43d5f9c5fb2664e59e4a89e24a776afc83a5 Mon Sep 17 00:00:00 2001 From: Fabien-Jrt <67684689+Fabien-jrt@users.noreply.github.com> Date: Sun, 24 Jul 2022 13:08:18 +0200 Subject: [PATCH 070/127] doc: Add resticprofile to scheduling section --- doc/040_backup.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/040_backup.rst b/doc/040_backup.rst index bbb9df892..c09307da0 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -533,8 +533,11 @@ Restic does not have a built-in way of scheduling backups, as it's a tool that runs when executed rather than a daemon. There are plenty of different ways to schedule backup runs on various different platforms, e.g. systemd and cron on Linux/BSD and Task Scheduler in Windows, depending on one's -needs and requirements. When scheduling restic to run recurringly, please -make sure to detect already running instances before starting the backup. +needs and requirements. If you don't want to implement your own scheduling, +you can use `resticprofile `__. + +When scheduling restic to run recurringly, please make sure to detect already +running instances before starting the backup. Space requirements ****************** From ebba233a3a86259e368f9826af269c70d8a27462 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 22 Apr 2023 12:32:57 +0200 Subject: [PATCH 071/127] backend/sema: rename constructor to NewBackend --- cmd/restic/global.go | 4 ++-- internal/backend/sema/backend.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 537fe02fe..43b8eb217 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -746,7 +746,7 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio } // wrap with debug logging and connection limiting - be = logger.New(sema.New(be)) + be = logger.New(sema.NewBackend(be)) // wrap backend if a test specified an inner hook if gopts.backendInnerTestHook != nil { @@ -821,5 +821,5 @@ func create(ctx context.Context, s string, opts options.Options) (restic.Backend return nil, err } - return logger.New(sema.New(be)), nil + return logger.New(sema.NewBackend(be)), nil } diff --git a/internal/backend/sema/backend.go b/internal/backend/sema/backend.go index 2294ef0b2..fc4a9dde5 100644 --- a/internal/backend/sema/backend.go +++ b/internal/backend/sema/backend.go @@ -18,8 +18,8 @@ type connectionLimitedBackend struct { sem semaphore } -// New creates a backend that limits the concurrent operations on the underlying backend -func New(be restic.Backend) restic.Backend { +// NewBackend creates a backend that limits the concurrent operations on the underlying backend +func NewBackend(be restic.Backend) restic.Backend { sem, err := newSemaphore(be.Connections()) if err != nil { panic(err) From 179e11c2ae4d305f55ede53d4e1b73f46aaf957e Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 22 Apr 2023 12:45:59 +0200 Subject: [PATCH 072/127] Increase timeouts for lock refresh tests When saving files to the local backend, in some cases the used fsync calls are slow enough to cause the tests to time out. Thus, increase the test timeouts as a stopgap measure until we can use the mem backend for these tests. --- cmd/restic/lock_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/restic/lock_test.go b/cmd/restic/lock_test.go index 1ad55bb4b..86daf83f0 100644 --- a/cmd/restic/lock_test.go +++ b/cmd/restic/lock_test.go @@ -156,8 +156,8 @@ func TestLockSuccessfulRefresh(t *testing.T) { t.Logf("test for successful lock refresh %v", time.Now()) // reduce locking intervals to be suitable for testing ri, rt := refreshInterval, refreshabilityTimeout - refreshInterval = 40 * time.Millisecond - refreshabilityTimeout = 200 * time.Millisecond + refreshInterval = 60 * time.Millisecond + refreshabilityTimeout = 500 * time.Millisecond defer func() { refreshInterval, refreshabilityTimeout = ri, rt }() @@ -189,7 +189,7 @@ func TestLockWaitTimeout(t *testing.T) { elock, _, err := lockRepoExclusive(context.TODO(), repo, env.gopts.RetryLock, env.gopts.JSON) test.OK(t, err) - retryLock := 100 * time.Millisecond + retryLock := 200 * time.Millisecond start := time.Now() lock, _, err := lockRepo(context.TODO(), repo, retryLock, env.gopts.JSON) @@ -199,7 +199,7 @@ func TestLockWaitTimeout(t *testing.T) { "create normal lock with exclusively locked repo didn't return an error") test.Assert(t, strings.Contains(err.Error(), "repository is already locked exclusively"), "create normal lock with exclusively locked repo didn't return the correct error") - test.Assert(t, retryLock <= duration && duration < retryLock+50*time.Millisecond, + test.Assert(t, retryLock <= duration && duration < retryLock*3/2, "create normal lock with exclusively locked repo didn't wait for the specified timeout") test.OK(t, lock.Unlock()) @@ -212,7 +212,7 @@ func TestLockWaitCancel(t *testing.T) { elock, _, err := lockRepoExclusive(context.TODO(), repo, env.gopts.RetryLock, env.gopts.JSON) test.OK(t, err) - retryLock := 100 * time.Millisecond + retryLock := 200 * time.Millisecond cancelAfter := 40 * time.Millisecond ctx, cancel := context.WithCancel(context.TODO()) @@ -226,7 +226,7 @@ func TestLockWaitCancel(t *testing.T) { "create normal lock with exclusively locked repo didn't return an error") test.Assert(t, strings.Contains(err.Error(), "context canceled"), "create normal lock with exclusively locked repo didn't return the correct error") - test.Assert(t, cancelAfter <= duration && duration < cancelAfter+50*time.Millisecond, + test.Assert(t, cancelAfter <= duration && duration < retryLock-10*time.Millisecond, "create normal lock with exclusively locked repo didn't return in time") test.OK(t, lock.Unlock()) @@ -240,7 +240,7 @@ func TestLockWaitSuccess(t *testing.T) { elock, _, err := lockRepoExclusive(context.TODO(), repo, env.gopts.RetryLock, env.gopts.JSON) test.OK(t, err) - retryLock := 100 * time.Millisecond + retryLock := 200 * time.Millisecond unlockAfter := 40 * time.Millisecond time.AfterFunc(unlockAfter, func() { From 831f593b875f922692cf1d5af6a120d9b89c4d0c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 22 Apr 2023 12:33:06 +0200 Subject: [PATCH 073/127] backend/sema: Add tests --- internal/backend/sema/backend_test.go | 180 ++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 internal/backend/sema/backend_test.go diff --git a/internal/backend/sema/backend_test.go b/internal/backend/sema/backend_test.go new file mode 100644 index 000000000..db9559840 --- /dev/null +++ b/internal/backend/sema/backend_test.go @@ -0,0 +1,180 @@ +package sema_test + +import ( + "context" + "io" + "sync/atomic" + "testing" + "time" + + "github.com/restic/restic/internal/backend/mock" + "github.com/restic/restic/internal/backend/sema" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" + "golang.org/x/sync/errgroup" +) + +func TestParameterValidationSave(t *testing.T) { + m := mock.NewBackend() + m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + return nil + } + be := sema.NewBackend(m) + + err := be.Save(context.TODO(), restic.Handle{}, nil) + test.Assert(t, err != nil, "Save() with invalid handle did not return an error") +} + +func TestParameterValidationLoad(t *testing.T) { + m := mock.NewBackend() + m.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + return io.NopCloser(nil), nil + } + + be := sema.NewBackend(m) + nilCb := func(rd io.Reader) error { return nil } + + err := be.Load(context.TODO(), restic.Handle{}, 10, 0, nilCb) + test.Assert(t, err != nil, "Load() with invalid handle did not return an error") + + h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + err = be.Load(context.TODO(), h, 10, -1, nilCb) + test.Assert(t, err != nil, "Save() with negative offset did not return an error") + err = be.Load(context.TODO(), h, -1, 0, nilCb) + test.Assert(t, err != nil, "Save() with negative length did not return an error") +} + +func TestParameterValidationStat(t *testing.T) { + m := mock.NewBackend() + m.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { + return restic.FileInfo{}, nil + } + be := sema.NewBackend(m) + + _, err := be.Stat(context.TODO(), restic.Handle{}) + test.Assert(t, err != nil, "Stat() with invalid handle did not return an error") +} + +func TestParameterValidationRemove(t *testing.T) { + m := mock.NewBackend() + m.RemoveFn = func(ctx context.Context, h restic.Handle) error { + return nil + } + be := sema.NewBackend(m) + + err := be.Remove(context.TODO(), restic.Handle{}) + test.Assert(t, err != nil, "Remove() with invalid handle did not return an error") +} + +func TestUnwrap(t *testing.T) { + m := mock.NewBackend() + be := sema.NewBackend(m) + + unwrapper := be.(restic.BackendUnwrapper) + test.Assert(t, unwrapper.Unwrap() == m, "Unwrap() returned wrong backend") +} + +func countingBlocker() (func(), func(int) int) { + ctr := int64(0) + blocker := make(chan struct{}) + + wait := func() { + // count how many goroutines were allowed by the semaphore + atomic.AddInt64(&ctr, 1) + // block until the test can retrieve the counter + <-blocker + } + + unblock := func(expected int) int { + // give goroutines enough time to block + var blocked int64 + for i := 0; i < 100 && blocked != int64(expected); i++ { + time.Sleep(100 * time.Microsecond) + blocked = atomic.LoadInt64(&ctr) + } + close(blocker) + return int(blocked) + } + + return wait, unblock +} + +func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(be restic.Backend) func() error, unblock func(int) int) { + expectBlocked := int(2) + + m := mock.NewBackend() + setup(m) + m.ConnectionsFn = func() uint { return uint(expectBlocked) } + be := sema.NewBackend(m) + + var wg errgroup.Group + for i := 0; i < int(expectBlocked+1); i++ { + wg.Go(handler(be)) + } + + blocked := unblock(expectBlocked) + test.Assert(t, blocked == expectBlocked, "Unexpected number of goroutines blocked: %v", blocked) + test.OK(t, wg.Wait()) +} + +func TestConcurrencyLimitSave(t *testing.T) { + wait, unblock := countingBlocker() + concurrencyTester(t, func(m *mock.Backend) { + m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + wait() + return nil + } + }, func(be restic.Backend) func() error { + return func() error { + h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + return be.Save(context.TODO(), h, nil) + } + }, unblock) +} + +func TestConcurrencyLimitLoad(t *testing.T) { + wait, unblock := countingBlocker() + concurrencyTester(t, func(m *mock.Backend) { + m.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + wait() + return io.NopCloser(nil), nil + } + }, func(be restic.Backend) func() error { + return func() error { + h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + nilCb := func(rd io.Reader) error { return nil } + return be.Load(context.TODO(), h, 10, 0, nilCb) + } + }, unblock) +} + +func TestConcurrencyLimitStat(t *testing.T) { + wait, unblock := countingBlocker() + concurrencyTester(t, func(m *mock.Backend) { + m.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { + wait() + return restic.FileInfo{}, nil + } + }, func(be restic.Backend) func() error { + return func() error { + h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + _, err := be.Stat(context.TODO(), h) + return err + } + }, unblock) +} + +func TestConcurrencyLimitDelete(t *testing.T) { + wait, unblock := countingBlocker() + concurrencyTester(t, func(m *mock.Backend) { + m.RemoveFn = func(ctx context.Context, h restic.Handle) error { + wait() + return nil + } + }, func(be restic.Backend) func() error { + return func() error { + h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + return be.Remove(context.TODO(), h) + } + }, unblock) +} From 78a1757e5ad57b842ada4b7d154cb2eb0ff42b30 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 10 Feb 2023 22:39:40 +0100 Subject: [PATCH 074/127] Cancel current command if cache becomes unusable If the cache suddenly disappears, the current command will now fail. --- changelog/unreleased/pull-4166 | 7 +++++++ internal/cache/backend.go | 35 ++++++++++++++++++---------------- internal/cache/file_test.go | 18 +++++++++++++++++ 3 files changed, 44 insertions(+), 16 deletions(-) create mode 100644 changelog/unreleased/pull-4166 diff --git a/changelog/unreleased/pull-4166 b/changelog/unreleased/pull-4166 new file mode 100644 index 000000000..6714fdf7f --- /dev/null +++ b/changelog/unreleased/pull-4166 @@ -0,0 +1,7 @@ +Enhancement: Cancel current command if cache becomes unusable + +If the cache directory was removed or ran out of space while restic was +running, this caused further caching attempts to fail and drastically slow down +the command execution. Now, the currently running command is canceled instead. + +https://github.com/restic/restic/pull/4166 diff --git a/internal/cache/backend.go b/internal/cache/backend.go index 08ec1facd..7c9df0f8e 100644 --- a/internal/cache/backend.go +++ b/internal/cache/backend.go @@ -83,7 +83,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea if err != nil { debug.Log("unable to save %v to cache: %v", h, err) _ = b.Cache.remove(h) - return nil + return err } return nil @@ -106,11 +106,19 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error { return nil } + defer func() { + // signal other waiting goroutines that the file may now be cached + close(finish) + + // remove the finish channel from the map + b.inProgressMutex.Lock() + delete(b.inProgress, h) + b.inProgressMutex.Unlock() + }() + // test again, maybe the file was cached in the meantime if !b.Cache.Has(h) { - // nope, it's still not in the cache, pull it from the repo and save it - err := b.Backend.Load(ctx, h, 0, 0, func(rd io.Reader) error { return b.Cache.Save(h, rd) }) @@ -118,16 +126,9 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error { // try to remove from the cache, ignore errors _ = b.Cache.remove(h) } + return err } - // signal other waiting goroutines that the file may now be cached - close(finish) - - // remove the finish channel from the map - b.inProgressMutex.Lock() - delete(b.inProgress, h) - b.inProgressMutex.Unlock() - return nil } @@ -178,11 +179,13 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset debug.Log("auto-store %v in the cache", h) err = b.cacheFile(ctx, h) - if err == nil { - inCache, err = b.loadFromCache(ctx, h, length, offset, consumer) - if inCache { - return err - } + if err != nil { + return err + } + + inCache, err = b.loadFromCache(ctx, h, length, offset, consumer) + if inCache { + return err } debug.Log("error caching %v: %v, falling back to backend", h, err) diff --git a/internal/cache/file_test.go b/internal/cache/file_test.go index 111a2430f..335e78aba 100644 --- a/internal/cache/file_test.go +++ b/internal/cache/file_test.go @@ -11,8 +11,10 @@ import ( "time" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" + rtest "github.com/restic/restic/internal/test" "golang.org/x/sync/errgroup" ) @@ -266,3 +268,19 @@ func TestFileSaveConcurrent(t *testing.T) { saved := load(t, c, h) test.Equals(t, data, saved) } + +func TestFileSaveAfterDamage(t *testing.T) { + c := TestNewCache(t) + rtest.OK(t, fs.RemoveAll(c.path)) + + // save a few bytes of data in the cache + data := test.Random(123456789, 42) + id := restic.Hash(data) + h := restic.Handle{ + Type: restic.PackFile, + Name: id.String(), + } + if err := c.Save(h, bytes.NewReader(data)); err == nil { + t.Fatal("Missing error when saving to deleted cache directory") + } +} From 94cbc6392dc0cc78ed70cf3639bb0dc09ea32714 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 23 Apr 2023 11:33:21 +0200 Subject: [PATCH 075/127] restore: slightly reduce memory usage while restoring files The information which target files are contained in a pack file is no longer necessary after processing a pack. --- internal/restorer/filerestorer.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index 75a19b4fb..3bb7489ba 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -181,6 +181,8 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { wg.Go(func() error { for _, id := range packOrder { pack := packs[id] + // allow garbage collection of packInfo + delete(packs, id) select { case <-ctx.Done(): return ctx.Err() From cdb0fb9c0645d098260667c4d3792302c77eb2d0 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 23 Apr 2023 11:38:06 +0200 Subject: [PATCH 076/127] tweak debug logs --- internal/archiver/tree_saver.go | 3 ++- internal/restic/parallel.go | 2 +- internal/walker/rewriter.go | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/archiver/tree_saver.go b/internal/archiver/tree_saver.go index d25781b03..a7dae3873 100644 --- a/internal/archiver/tree_saver.go +++ b/internal/archiver/tree_saver.go @@ -105,14 +105,15 @@ func (s *TreeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, I continue } - debug.Log("insert %v", fnr.node.Name) err := builder.AddNode(fnr.node) if err != nil && errors.Is(err, restic.ErrTreeNotOrdered) && lastNode != nil && fnr.node.Equals(*lastNode) { + debug.Log("insert %v failed: %v", fnr.node.Name, err) // ignore error if an _identical_ node already exists, but nevertheless issue a warning _ = s.errFn(fnr.target, err) err = nil } if err != nil { + debug.Log("insert %v failed: %v", fnr.node.Name, err) return nil, stats, err } lastNode = fnr.node diff --git a/internal/restic/parallel.go b/internal/restic/parallel.go index df160f018..34a2a019c 100644 --- a/internal/restic/parallel.go +++ b/internal/restic/parallel.go @@ -41,7 +41,7 @@ func ParallelList(ctx context.Context, r Lister, t FileType, parallelism uint, f // a worker receives an index ID from ch, loads the index, and sends it to indexCh worker := func() error { for fi := range ch { - debug.Log("worker got file %v", fi.ID.Str()) + debug.Log("worker got file %v/%v", t, fi.ID.Str()) err := fn(ctx, fi.ID, fi.Size) if err != nil { return err diff --git a/internal/walker/rewriter.go b/internal/walker/rewriter.go index 6f063831e..cd05f69f5 100644 --- a/internal/walker/rewriter.go +++ b/internal/walker/rewriter.go @@ -37,7 +37,7 @@ func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID return restic.ID{}, err } if nodeID != testID { - return restic.ID{}, fmt.Errorf("cannot encode tree at %q without loosing information", nodepath) + return restic.ID{}, fmt.Errorf("cannot encode tree at %q without losing information", nodepath) } debug.Log("filterTree: %s, nodeId: %s\n", nodepath, nodeID.Str()) From f342db7666fd49acee80e1c5c30acff04aa0c82c Mon Sep 17 00:00:00 2001 From: greatroar <61184462+greatroar@users.noreply.github.com> Date: Sat, 11 Feb 2023 14:51:58 +0100 Subject: [PATCH 077/127] ui/termstatus: Quote funny filenames Fixes #2260, #4191. --- changelog/unreleased/issue-2260 | 13 +++++++++++ internal/ui/backup/text.go | 2 ++ internal/ui/termstatus/status.go | 29 ++++++++++++++++++------ internal/ui/termstatus/status_test.go | 32 ++++++++++++++++++++++++++- 4 files changed, 68 insertions(+), 8 deletions(-) create mode 100644 changelog/unreleased/issue-2260 diff --git a/changelog/unreleased/issue-2260 b/changelog/unreleased/issue-2260 new file mode 100644 index 000000000..96c79a035 --- /dev/null +++ b/changelog/unreleased/issue-2260 @@ -0,0 +1,13 @@ +Bugfix: Exotic filenames no longer break restic backup's status output + +Restic backup shows the names of files that it is working on. In previous +versions of restic, those names were printed without first sanitizing them, +so that filenames containing newlines or terminal control characters could +mess up restic backup's output or even change the state of a terminal. + +Filenames are now checked and quoted if they contain non-printable or +non-Unicode characters. + +https://github.com/restic/restic/issues/2260 +https://github.com/restic/restic/issues/4191 +https://github.com/restic/restic/pull/4192 diff --git a/internal/ui/backup/text.go b/internal/ui/backup/text.go index 0c5f897dd..acb2a8d3a 100644 --- a/internal/ui/backup/text.go +++ b/internal/ui/backup/text.go @@ -86,6 +86,8 @@ func (b *TextProgress) Error(item string, err error) error { // CompleteItem is the status callback function for the archiver when a // file/dir has been saved successfully. func (b *TextProgress) CompleteItem(messageType, item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration) { + item = termstatus.Quote(item) + switch messageType { case "dir new": b.VV("new %v, saved in %.3fs (%v added, %v stored, %v metadata)", diff --git a/internal/ui/termstatus/status.go b/internal/ui/termstatus/status.go index fdc7e14f6..a1b7a5fcc 100644 --- a/internal/ui/termstatus/status.go +++ b/internal/ui/termstatus/status.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "os" + "strconv" "strings" "unicode" @@ -325,6 +326,7 @@ func wideRune(r rune) bool { } // SetStatus updates the status lines. +// The lines should not contain newlines; this method adds them. func (t *Terminal) SetStatus(lines []string) { if len(lines) == 0 { return @@ -341,21 +343,34 @@ func (t *Terminal) SetStatus(lines []string) { } } - // make sure that all lines have a line break and are not too long + // Sanitize lines and truncate them if they're too long. for i, line := range lines { - line = strings.TrimRight(line, "\n") + line = Quote(line) if width > 0 { line = Truncate(line, width-2) } - lines[i] = line + "\n" + if i < len(lines)-1 { // Last line gets no line break. + lines[i] = line + "\n" + } } - // make sure the last line does not have a line break - last := len(lines) - 1 - lines[last] = strings.TrimRight(lines[last], "\n") - select { case t.status <- status{lines: lines}: case <-t.closed: } } + +// Quote lines with funny characters in them, meaning control chars, newlines, +// tabs, anything else non-printable and invalid UTF-8. +// +// This is intended to produce a string that does not mess up the terminal +// rather than produce an unambiguous quoted string. +func Quote(line string) string { + for _, r := range line { + // The replacement character usually means the input is not UTF-8. + if r == unicode.ReplacementChar || !unicode.IsPrint(r) { + return strconv.Quote(line) + } + } + return line +} diff --git a/internal/ui/termstatus/status_test.go b/internal/ui/termstatus/status_test.go index ce18f42e6..40a908deb 100644 --- a/internal/ui/termstatus/status_test.go +++ b/internal/ui/termstatus/status_test.go @@ -1,6 +1,36 @@ package termstatus -import "testing" +import ( + "strconv" + "testing" + + rtest "github.com/restic/restic/internal/test" +) + +func TestQuote(t *testing.T) { + for _, c := range []struct { + in string + needQuote bool + }{ + {"foo.bar/baz", false}, + {"föó_bàŕ-bãẑ", false}, + {" foo ", false}, + {"foo bar", false}, + {"foo\nbar", true}, + {"foo\rbar", true}, + {"foo\abar", true}, + {"\xff", true}, + {`c:\foo\bar`, false}, + // Issue #2260: terminal control characters. + {"\x1bm_red_is_beautiful", true}, + } { + if c.needQuote { + rtest.Equals(t, strconv.Quote(c.in), Quote(c.in)) + } else { + rtest.Equals(t, c.in, Quote(c.in)) + } + } +} func TestTruncate(t *testing.T) { var tests = []struct { From 1e6e9f9bd00362d572c5e3f8f99dc22b82c8a7a8 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 23 Apr 2023 12:40:29 +0200 Subject: [PATCH 078/127] tweak changelogs --- changelog/unreleased/issue-2260 | 4 ++-- changelog/unreleased/issue-4211 | 8 ++++---- changelog/unreleased/issue-4239 | 6 +++--- changelog/unreleased/issue-4253 | 13 ++++++------- changelog/unreleased/pull-4180 | 2 +- changelog/unreleased/pull-4219 | 2 +- 6 files changed, 17 insertions(+), 18 deletions(-) diff --git a/changelog/unreleased/issue-2260 b/changelog/unreleased/issue-2260 index 96c79a035..795354c22 100644 --- a/changelog/unreleased/issue-2260 +++ b/changelog/unreleased/issue-2260 @@ -1,9 +1,9 @@ Bugfix: Exotic filenames no longer break restic backup's status output -Restic backup shows the names of files that it is working on. In previous +Restic `backup` shows the names of files that it is working on. In previous versions of restic, those names were printed without first sanitizing them, so that filenames containing newlines or terminal control characters could -mess up restic backup's output or even change the state of a terminal. +mangle the status output or even change the state of a terminal. Filenames are now checked and quoted if they contain non-printable or non-Unicode characters. diff --git a/changelog/unreleased/issue-4211 b/changelog/unreleased/issue-4211 index 45b7aee83..214418986 100644 --- a/changelog/unreleased/issue-4211 +++ b/changelog/unreleased/issue-4211 @@ -1,8 +1,8 @@ -Bugfix: Restic dump now interprets --host and --path correctly +Bugfix: Make `dump` interpret --host and --path correctly -Restic dump previously confused its --host= and --path= -options: it looked for snapshots with paths called from hosts -called . It now treats the options as intended. +A regression in restic 0.15.0 caused `dump` to confuse its `--host=` and +`--path=` options: it looked for snapshots with paths called `` +from hosts called ``. It now treats the options as intended. https://github.com/restic/restic/issues/4211 https://github.com/restic/restic/pull/4212 diff --git a/changelog/unreleased/issue-4239 b/changelog/unreleased/issue-4239 index 247f3d9ed..24d8bfe4e 100644 --- a/changelog/unreleased/issue-4239 +++ b/changelog/unreleased/issue-4239 @@ -1,8 +1,8 @@ Bugfix: Correct number of blocks reported in mount point -Restic mount points incorrectly reported the number of 512-byte (POSIX -standard) blocks for files and links, due to a rounding bug. In particular, -empty files were reported as taking one block instead of zero. +Restic mount points reported and incorrect number of 512-byte (POSIX standard) +blocks for files and links due to a rounding bug. In particular, empty files +were reported as taking one block instead of zero. The rounding is now fixed: the number of blocks reported is the file size (or link target size), divided by 512 and rounded up to a whole number. diff --git a/changelog/unreleased/issue-4253 b/changelog/unreleased/issue-4253 index 2471eab0b..1226e6e6d 100644 --- a/changelog/unreleased/issue-4253 +++ b/changelog/unreleased/issue-4253 @@ -1,11 +1,10 @@ -Bugfix: Mount command should no longer create spurious filesystem loops +Bugfix: `mount` should no longer create spurious filesystem loops -When a backup contains a directory that has the same name as its parent, -say, a/b/b, and the GNU find command were run on this backup in a restic -mount, find command would refuse to traverse the lowest "b" directory, -instead printing "File system loop detected". This is due to the way the -restic mount command generates inode numbers for directories in the mount -point. +When a backup contains a directory that has the same name as its parent, say +`a/b/b`, and the GNU `find` command were run on this backup in a restic mount, +`find` would refuse to traverse the lowest `b` directory, instead printing +`File system loop detected`. This is due to the way the restic mount command +generates inode numbers for directories in the mount point. The rule for generating these inode numbers was changed in 0.15.0. It has now been changed again to avoid this issue. A perfect rule does not exist, diff --git a/changelog/unreleased/pull-4180 b/changelog/unreleased/pull-4180 index ff43feb2b..054791e0f 100644 --- a/changelog/unreleased/pull-4180 +++ b/changelog/unreleased/pull-4180 @@ -1,5 +1,5 @@ Enhancement: Add release binaries for riscv64 architecture on Linux -We've added release binaries for riscv64 architecture on Linux. +We've added release binaries for the riscv64 architecture on Linux. https://github.com/restic/restic/pull/4180 diff --git a/changelog/unreleased/pull-4219 b/changelog/unreleased/pull-4219 index 7d20c3607..4f605bd1d 100644 --- a/changelog/unreleased/pull-4219 +++ b/changelog/unreleased/pull-4219 @@ -1,5 +1,5 @@ Enhancement: Upgrade Minio to 7.0.49 -Upgraded to allow use of the ap-southeast-4 region (Melbourne) +Upgraded to allow use of the ap-southeast-4 region (Melbourne). https://github.com/restic/restic/pull/4219 From 306a29980a272891387d53ede50a2f83df7dcfda Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 23 Apr 2023 14:56:36 +0200 Subject: [PATCH 079/127] Print stacktrace in SIGINT handler if RESTIC_DEBUG_STACKTRACE_SIGINT set The builtin mechanism to capture a stacktrace in Go is to send a SIGQUIT to the running process. However, this mechanism is not avaiable on Windows. Thus, tweak the SIGINT handler to dump a stacktrace if the environment variable `RESTIC_DEBUG_STACKTRACE_SIGINT` is set. --- CONTRIBUTING.md | 10 ++++++++++ cmd/restic/cleanup.go | 6 ++++++ internal/debug/stacktrace.go | 15 +++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 internal/debug/stacktrace.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4b4be0757..36a7c0695 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,6 +58,16 @@ Please be aware that the debug log file will contain potentially sensitive things like file and directory names, so please either redact it before uploading it somewhere or post only the parts that are really relevant. +If restic gets stuck, please also include a stacktrace in the description. +On non-Windows systems, you can send a SIGQUIT signal to restic or press +`Ctrl-\` to achieve the same result. This causes restic to print a stacktrace +and then exit immediatelly. This will not damage your repository, however, +it might be necessary to manually clean up stale lock files using +`restic unlock`. + +On Windows, please set the environment variable `RESTIC_DEBUG_STACKTRACE_SIGINT` +to `true` and press `Ctrl-C` to create a stacktrace. + Development Environment ======================= diff --git a/cmd/restic/cleanup.go b/cmd/restic/cleanup.go index 967957106..75933fe96 100644 --- a/cmd/restic/cleanup.go +++ b/cmd/restic/cleanup.go @@ -62,6 +62,12 @@ func CleanupHandler(c <-chan os.Signal) { debug.Log("signal %v received, cleaning up", s) Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s) + if val, _ := os.LookupEnv("RESTIC_DEBUG_STACKTRACE_SIGINT"); val != "" { + _, _ = os.Stderr.WriteString("\n--- STACKTRACE START ---\n\n") + _, _ = os.Stderr.WriteString(debug.DumpStacktrace()) + _, _ = os.Stderr.WriteString("\n--- STACKTRACE END ---\n") + } + code := 0 if s == syscall.SIGINT { diff --git a/internal/debug/stacktrace.go b/internal/debug/stacktrace.go new file mode 100644 index 000000000..a8db83160 --- /dev/null +++ b/internal/debug/stacktrace.go @@ -0,0 +1,15 @@ +package debug + +import "runtime" + +func DumpStacktrace() string { + buf := make([]byte, 128*1024) + + for { + l := runtime.Stack(buf, true) + if l < len(buf) { + return string(buf[:l]) + } + buf = make([]byte, len(buf)*2) + } +} From 0bac935dac8058a7a3ea85055544116f28f00c24 Mon Sep 17 00:00:00 2001 From: "Leo R. Lundgren" Date: Sun, 23 Apr 2023 22:09:52 +0200 Subject: [PATCH 080/127] doc: Polish changelogs --- changelog/unreleased/issue-2260 | 9 ++++----- changelog/unreleased/issue-4211 | 2 +- changelog/unreleased/issue-4239 | 4 ++-- changelog/unreleased/issue-4253 | 7 ++++--- changelog/unreleased/issue-4275 | 2 +- changelog/unreleased/pull-4180 | 3 ++- changelog/unreleased/pull-4219 | 4 ++-- 7 files changed, 16 insertions(+), 15 deletions(-) diff --git a/changelog/unreleased/issue-2260 b/changelog/unreleased/issue-2260 index 795354c22..da4fe8e99 100644 --- a/changelog/unreleased/issue-2260 +++ b/changelog/unreleased/issue-2260 @@ -1,9 +1,8 @@ -Bugfix: Exotic filenames no longer break restic backup's status output +Bugfix: Sanitize filenames printed by `backup` during processing -Restic `backup` shows the names of files that it is working on. In previous -versions of restic, those names were printed without first sanitizing them, -so that filenames containing newlines or terminal control characters could -mangle the status output or even change the state of a terminal. +The `backup` command would previously not sanitize the filenames it printed +during processing, potentially causing newlines or terminal control characters +to mangle the status output or even change the state of a terminal. Filenames are now checked and quoted if they contain non-printable or non-Unicode characters. diff --git a/changelog/unreleased/issue-4211 b/changelog/unreleased/issue-4211 index 214418986..0d499977c 100644 --- a/changelog/unreleased/issue-4211 +++ b/changelog/unreleased/issue-4211 @@ -1,4 +1,4 @@ -Bugfix: Make `dump` interpret --host and --path correctly +Bugfix: Make `dump` interpret `--host` and `--path` correctly A regression in restic 0.15.0 caused `dump` to confuse its `--host=` and `--path=` options: it looked for snapshots with paths called `` diff --git a/changelog/unreleased/issue-4239 b/changelog/unreleased/issue-4239 index 24d8bfe4e..43d099e24 100644 --- a/changelog/unreleased/issue-4239 +++ b/changelog/unreleased/issue-4239 @@ -1,11 +1,11 @@ Bugfix: Correct number of blocks reported in mount point -Restic mount points reported and incorrect number of 512-byte (POSIX standard) +Restic mount points reported an incorrect number of 512-byte (POSIX standard) blocks for files and links due to a rounding bug. In particular, empty files were reported as taking one block instead of zero. The rounding is now fixed: the number of blocks reported is the file size -(or link target size), divided by 512 and rounded up to a whole number. +(or link target size) divided by 512 and rounded up to a whole number. https://github.com/restic/restic/issues/4239 https://github.com/restic/restic/pull/4240 diff --git a/changelog/unreleased/issue-4253 b/changelog/unreleased/issue-4253 index 1226e6e6d..d9109f988 100644 --- a/changelog/unreleased/issue-4253 +++ b/changelog/unreleased/issue-4253 @@ -1,14 +1,15 @@ -Bugfix: `mount` should no longer create spurious filesystem loops +Bugfix: Minimize risk of spurious filesystem loops with `mount` When a backup contains a directory that has the same name as its parent, say -`a/b/b`, and the GNU `find` command were run on this backup in a restic mount, +`a/b/b`, and the GNU `find` command was run on this backup in a restic mount, `find` would refuse to traverse the lowest `b` directory, instead printing -`File system loop detected`. This is due to the way the restic mount command +`File system loop detected`. This was due to the way the restic mount command generates inode numbers for directories in the mount point. The rule for generating these inode numbers was changed in 0.15.0. It has now been changed again to avoid this issue. A perfect rule does not exist, but the probability of this behavior occurring is now extremely small. + When it does occur, the mount point is not broken, and scripts that traverse the mount point should work as long as they don't rely on inode numbers for detecting filesystem loops. diff --git a/changelog/unreleased/issue-4275 b/changelog/unreleased/issue-4275 index 3fbe093fb..944797b85 100644 --- a/changelog/unreleased/issue-4275 +++ b/changelog/unreleased/issue-4275 @@ -1,4 +1,4 @@ -Security: Update golang.org/x/net to address CVE-2022-41723 +Security: Update golang.org/x/net to address CVE-2022-41723 https://github.com/restic/restic/issues/4275 https://github.com/restic/restic/pull/4213 diff --git a/changelog/unreleased/pull-4180 b/changelog/unreleased/pull-4180 index 054791e0f..511974963 100644 --- a/changelog/unreleased/pull-4180 +++ b/changelog/unreleased/pull-4180 @@ -1,5 +1,6 @@ Enhancement: Add release binaries for riscv64 architecture on Linux -We've added release binaries for the riscv64 architecture on Linux. +Builds for the `riscv64` architecture on Linux are now included in the +release binaries. https://github.com/restic/restic/pull/4180 diff --git a/changelog/unreleased/pull-4219 b/changelog/unreleased/pull-4219 index 4f605bd1d..25da7058b 100644 --- a/changelog/unreleased/pull-4219 +++ b/changelog/unreleased/pull-4219 @@ -1,5 +1,5 @@ -Enhancement: Upgrade Minio to 7.0.49 +Enhancement: Upgrade Minio to version 7.0.49 -Upgraded to allow use of the ap-southeast-4 region (Melbourne). +The upgraded version now allows use of the `ap-southeast-4` region (Melbourne). https://github.com/restic/restic/pull/4219 From 0aaa4e6cbecd2b46f9be0ad8c70fc9b2f3419677 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 24 Apr 2023 20:28:24 +0200 Subject: [PATCH 081/127] Prepare changelog for 0.15.2 --- changelog/{unreleased => 0.15.2_2023-04-24}/issue-2260 | 0 changelog/{unreleased => 0.15.2_2023-04-24}/issue-4211 | 0 changelog/{unreleased => 0.15.2_2023-04-24}/issue-4239 | 0 changelog/{unreleased => 0.15.2_2023-04-24}/issue-4253 | 0 changelog/{unreleased => 0.15.2_2023-04-24}/issue-4275 | 0 changelog/{unreleased => 0.15.2_2023-04-24}/pull-4180 | 0 changelog/{unreleased => 0.15.2_2023-04-24}/pull-4219 | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename changelog/{unreleased => 0.15.2_2023-04-24}/issue-2260 (100%) rename changelog/{unreleased => 0.15.2_2023-04-24}/issue-4211 (100%) rename changelog/{unreleased => 0.15.2_2023-04-24}/issue-4239 (100%) rename changelog/{unreleased => 0.15.2_2023-04-24}/issue-4253 (100%) rename changelog/{unreleased => 0.15.2_2023-04-24}/issue-4275 (100%) rename changelog/{unreleased => 0.15.2_2023-04-24}/pull-4180 (100%) rename changelog/{unreleased => 0.15.2_2023-04-24}/pull-4219 (100%) diff --git a/changelog/unreleased/issue-2260 b/changelog/0.15.2_2023-04-24/issue-2260 similarity index 100% rename from changelog/unreleased/issue-2260 rename to changelog/0.15.2_2023-04-24/issue-2260 diff --git a/changelog/unreleased/issue-4211 b/changelog/0.15.2_2023-04-24/issue-4211 similarity index 100% rename from changelog/unreleased/issue-4211 rename to changelog/0.15.2_2023-04-24/issue-4211 diff --git a/changelog/unreleased/issue-4239 b/changelog/0.15.2_2023-04-24/issue-4239 similarity index 100% rename from changelog/unreleased/issue-4239 rename to changelog/0.15.2_2023-04-24/issue-4239 diff --git a/changelog/unreleased/issue-4253 b/changelog/0.15.2_2023-04-24/issue-4253 similarity index 100% rename from changelog/unreleased/issue-4253 rename to changelog/0.15.2_2023-04-24/issue-4253 diff --git a/changelog/unreleased/issue-4275 b/changelog/0.15.2_2023-04-24/issue-4275 similarity index 100% rename from changelog/unreleased/issue-4275 rename to changelog/0.15.2_2023-04-24/issue-4275 diff --git a/changelog/unreleased/pull-4180 b/changelog/0.15.2_2023-04-24/pull-4180 similarity index 100% rename from changelog/unreleased/pull-4180 rename to changelog/0.15.2_2023-04-24/pull-4180 diff --git a/changelog/unreleased/pull-4219 b/changelog/0.15.2_2023-04-24/pull-4219 similarity index 100% rename from changelog/unreleased/pull-4219 rename to changelog/0.15.2_2023-04-24/pull-4219 From 1f3f042f32caff8a4e74c5da3545335133d5f8c1 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 24 Apr 2023 20:28:24 +0200 Subject: [PATCH 082/127] Generate CHANGELOG.md for 0.15.2 --- CHANGELOG.md | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e5635404..a502d49e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,92 @@ +Changelog for restic 0.15.2 (2023-04-24) +======================================= + +The following sections list the changes in restic 0.15.2 relevant to +restic users. The changes are ordered by importance. + +Summary +------- + + * Sec #4275: Update golang.org/x/net to address CVE-2022-41723 + * Fix #2260: Sanitize filenames printed by `backup` during processing + * Fix #4211: Make `dump` interpret `--host` and `--path` correctly + * Fix #4239: Correct number of blocks reported in mount point + * Fix #4253: Minimize risk of spurious filesystem loops with `mount` + * Enh #4180: Add release binaries for riscv64 architecture on Linux + * Enh #4219: Upgrade Minio to version 7.0.49 + +Details +------- + + * Security #4275: Update golang.org/x/net to address CVE-2022-41723 + + https://github.com/restic/restic/issues/4275 + https://github.com/restic/restic/pull/4213 + + * Bugfix #2260: Sanitize filenames printed by `backup` during processing + + The `backup` command would previously not sanitize the filenames it printed during + processing, potentially causing newlines or terminal control characters to mangle the + status output or even change the state of a terminal. + + Filenames are now checked and quoted if they contain non-printable or non-Unicode + characters. + + https://github.com/restic/restic/issues/2260 + https://github.com/restic/restic/issues/4191 + https://github.com/restic/restic/pull/4192 + + * Bugfix #4211: Make `dump` interpret `--host` and `--path` correctly + + A regression in restic 0.15.0 caused `dump` to confuse its `--host=` and + `--path=` options: it looked for snapshots with paths called `` from hosts + called ``. It now treats the options as intended. + + https://github.com/restic/restic/issues/4211 + https://github.com/restic/restic/pull/4212 + + * Bugfix #4239: Correct number of blocks reported in mount point + + Restic mount points reported an incorrect number of 512-byte (POSIX standard) blocks for + files and links due to a rounding bug. In particular, empty files were reported as taking one + block instead of zero. + + The rounding is now fixed: the number of blocks reported is the file size (or link target size) + divided by 512 and rounded up to a whole number. + + https://github.com/restic/restic/issues/4239 + https://github.com/restic/restic/pull/4240 + + * Bugfix #4253: Minimize risk of spurious filesystem loops with `mount` + + When a backup contains a directory that has the same name as its parent, say `a/b/b`, and the GNU + `find` command was run on this backup in a restic mount, `find` would refuse to traverse the + lowest `b` directory, instead printing `File system loop detected`. This was due to the way the + restic mount command generates inode numbers for directories in the mount point. + + The rule for generating these inode numbers was changed in 0.15.0. It has now been changed again + to avoid this issue. A perfect rule does not exist, but the probability of this behavior + occurring is now extremely small. + + When it does occur, the mount point is not broken, and scripts that traverse the mount point + should work as long as they don't rely on inode numbers for detecting filesystem loops. + + https://github.com/restic/restic/issues/4253 + https://github.com/restic/restic/pull/4255 + + * Enhancement #4180: Add release binaries for riscv64 architecture on Linux + + Builds for the `riscv64` architecture on Linux are now included in the release binaries. + + https://github.com/restic/restic/pull/4180 + + * Enhancement #4219: Upgrade Minio to version 7.0.49 + + The upgraded version now allows use of the `ap-southeast-4` region (Melbourne). + + https://github.com/restic/restic/pull/4219 + + Changelog for restic 0.15.1 (2023-01-30) ======================================= From db6b4f8912519e908a9b53609019ec7a075f99dd Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 24 Apr 2023 20:28:37 +0200 Subject: [PATCH 083/127] Update manpages and auto-completion --- doc/man/restic-backup.1 | 2 +- doc/man/restic-cache.1 | 2 +- doc/man/restic-cat.1 | 2 +- doc/man/restic-check.1 | 2 +- doc/man/restic-copy.1 | 2 +- doc/man/restic-diff.1 | 2 +- doc/man/restic-dump.1 | 2 +- doc/man/restic-find.1 | 2 +- doc/man/restic-forget.1 | 2 +- doc/man/restic-generate.1 | 2 +- doc/man/restic-init.1 | 2 +- doc/man/restic-key.1 | 2 +- doc/man/restic-list.1 | 2 +- doc/man/restic-ls.1 | 2 +- doc/man/restic-migrate.1 | 2 +- doc/man/restic-mount.1 | 2 +- doc/man/restic-prune.1 | 2 +- doc/man/restic-rebuild-index.1 | 2 +- doc/man/restic-recover.1 | 2 +- doc/man/restic-restore.1 | 2 +- doc/man/restic-rewrite.1 | 2 +- doc/man/restic-self-update.1 | 2 +- doc/man/restic-snapshots.1 | 2 +- doc/man/restic-stats.1 | 2 +- doc/man/restic-tag.1 | 2 +- doc/man/restic-unlock.1 | 2 +- doc/man/restic-version.1 | 2 +- doc/man/restic.1 | 2 +- 28 files changed, 28 insertions(+), 28 deletions(-) diff --git a/doc/man/restic-backup.1 b/doc/man/restic-backup.1 index 2598678d0..4297c3b8e 100644 --- a/doc/man/restic-backup.1 +++ b/doc/man/restic-backup.1 @@ -205,7 +205,7 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-cache.1 b/doc/man/restic-cache.1 index 302bcb01e..3552fb1dc 100644 --- a/doc/man/restic-cache.1 +++ b/doc/man/restic-cache.1 @@ -118,7 +118,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-cat.1 b/doc/man/restic-cat.1 index 1fb7dd45f..2e787fa06 100644 --- a/doc/man/restic-cat.1 +++ b/doc/man/restic-cat.1 @@ -106,7 +106,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-check.1 b/doc/man/restic-check.1 index e08b83d46..e641fc2b5 100644 --- a/doc/man/restic-check.1 +++ b/doc/man/restic-check.1 @@ -123,7 +123,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-copy.1 b/doc/man/restic-copy.1 index 07dcfe957..53badecc9 100644 --- a/doc/man/restic-copy.1 +++ b/doc/man/restic-copy.1 @@ -147,7 +147,7 @@ new destination repository using the "init" command. .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-diff.1 b/doc/man/restic-diff.1 index f0707a257..31c34dc8a 100644 --- a/doc/man/restic-diff.1 +++ b/doc/man/restic-diff.1 @@ -126,7 +126,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-dump.1 b/doc/man/restic-dump.1 index f9a2368bc..61b3b3ec8 100644 --- a/doc/man/restic-dump.1 +++ b/doc/man/restic-dump.1 @@ -129,7 +129,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-find.1 b/doc/man/restic-find.1 index 4f5bdd4e3..9fa4dd811 100644 --- a/doc/man/restic-find.1 +++ b/doc/man/restic-find.1 @@ -151,7 +151,7 @@ It can also be used to search for restic blobs or trees for troubleshooting. .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH EXAMPLE diff --git a/doc/man/restic-forget.1 b/doc/man/restic-forget.1 index f46d05736..d8a69856e 100644 --- a/doc/man/restic-forget.1 +++ b/doc/man/restic-forget.1 @@ -217,7 +217,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-generate.1 b/doc/man/restic-generate.1 index e3733ce60..6b54ebfca 100644 --- a/doc/man/restic-generate.1 +++ b/doc/man/restic-generate.1 @@ -127,7 +127,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-init.1 b/doc/man/restic-init.1 index 80edf5362..194f31756 100644 --- a/doc/man/restic-init.1 +++ b/doc/man/restic-init.1 @@ -134,7 +134,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-key.1 b/doc/man/restic-key.1 index ff6ab4fd0..4163cefa5 100644 --- a/doc/man/restic-key.1 +++ b/doc/man/restic-key.1 @@ -118,7 +118,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-list.1 b/doc/man/restic-list.1 index e2f878c76..6683e2c47 100644 --- a/doc/man/restic-list.1 +++ b/doc/man/restic-list.1 @@ -106,7 +106,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-ls.1 b/doc/man/restic-ls.1 index afd72ff71..a16716434 100644 --- a/doc/man/restic-ls.1 +++ b/doc/man/restic-ls.1 @@ -141,7 +141,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-migrate.1 b/doc/man/restic-migrate.1 index ee4d44e71..d8127090e 100644 --- a/doc/man/restic-migrate.1 +++ b/doc/man/restic-migrate.1 @@ -112,7 +112,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-mount.1 b/doc/man/restic-mount.1 index da38ae451..ce4f893a7 100644 --- a/doc/man/restic-mount.1 +++ b/doc/man/restic-mount.1 @@ -190,7 +190,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-prune.1 b/doc/man/restic-prune.1 index 88c03f72a..197cb1130 100644 --- a/doc/man/restic-prune.1 +++ b/doc/man/restic-prune.1 @@ -135,7 +135,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-rebuild-index.1 b/doc/man/restic-rebuild-index.1 index 3be67e79e..18878b66f 100644 --- a/doc/man/restic-rebuild-index.1 +++ b/doc/man/restic-rebuild-index.1 @@ -111,7 +111,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-recover.1 b/doc/man/restic-recover.1 index 7415a1113..aa3441156 100644 --- a/doc/man/restic-recover.1 +++ b/doc/man/restic-recover.1 @@ -108,7 +108,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-restore.1 b/doc/man/restic-restore.1 index 2348f7478..39ff62059 100644 --- a/doc/man/restic-restore.1 +++ b/doc/man/restic-restore.1 @@ -151,7 +151,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-rewrite.1 b/doc/man/restic-rewrite.1 index 9f33bcb64..6edf51b95 100644 --- a/doc/man/restic-rewrite.1 +++ b/doc/man/restic-rewrite.1 @@ -159,7 +159,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-self-update.1 b/doc/man/restic-self-update.1 index 25f863396..e311b2277 100644 --- a/doc/man/restic-self-update.1 +++ b/doc/man/restic-self-update.1 @@ -113,7 +113,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-snapshots.1 b/doc/man/restic-snapshots.1 index 78cd664e3..d2dbf52ee 100644 --- a/doc/man/restic-snapshots.1 +++ b/doc/man/restic-snapshots.1 @@ -130,7 +130,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-stats.1 b/doc/man/restic-stats.1 index 6e3b9838b..694bde22d 100644 --- a/doc/man/restic-stats.1 +++ b/doc/man/restic-stats.1 @@ -152,7 +152,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-tag.1 b/doc/man/restic-tag.1 index 06bf25495..1ff0b4f78 100644 --- a/doc/man/restic-tag.1 +++ b/doc/man/restic-tag.1 @@ -137,7 +137,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-unlock.1 b/doc/man/restic-unlock.1 index c4ad7f050..e5b408915 100644 --- a/doc/man/restic-unlock.1 +++ b/doc/man/restic-unlock.1 @@ -110,7 +110,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic-version.1 b/doc/man/restic-version.1 index b410d1231..eca34d60a 100644 --- a/doc/man/restic-version.1 +++ b/doc/man/restic-version.1 @@ -107,7 +107,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO diff --git a/doc/man/restic.1 b/doc/man/restic.1 index 76602d02d..f76d16e38 100644 --- a/doc/man/restic.1 +++ b/doc/man/restic.1 @@ -100,7 +100,7 @@ directories in an encrypted repository stored on different backends. .PP \fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2) + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) .SH SEE ALSO From be8be3397c2ba4f3ec986998a5af4a5a4c309bc3 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 24 Apr 2023 20:28:37 +0200 Subject: [PATCH 084/127] Add version for 0.15.2 --- VERSION | 2 +- cmd/restic/global.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index e815b861f..4312e0d0c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.15.1 +0.15.2 diff --git a/cmd/restic/global.go b/cmd/restic/global.go index b32265275..206229d94 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -42,7 +42,7 @@ import ( "golang.org/x/term" ) -var version = "0.15.1" +var version = "0.15.2" // TimeFormat is the format used for all timestamps printed by restic. const TimeFormat = "2006-01-02 15:04:05" From ac7ac0cb9725e5abc7f7fe4191672337451f6ca9 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 24 Apr 2023 20:28:37 +0200 Subject: [PATCH 085/127] Set development version for 0.15.2 --- cmd/restic/global.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 206229d94..d8b8b6a96 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -42,7 +42,7 @@ import ( "golang.org/x/term" ) -var version = "0.15.2" +var version = "0.15.2-dev (compiled manually)" // TimeFormat is the format used for all timestamps printed by restic. const TimeFormat = "2006-01-02 15:04:05" From b50ff04cf3af4bee87bb38e95c3413d8bbf51690 Mon Sep 17 00:00:00 2001 From: greatroar <61184462+greatroar@users.noreply.github.com> Date: Thu, 27 Apr 2023 11:45:41 +0200 Subject: [PATCH 086/127] dump: Report filename with tar.ErrFieldTooLong Updates #4307. --- internal/dump/tar.go | 4 ++-- internal/dump/tar_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/internal/dump/tar.go b/internal/dump/tar.go index 6e87aabe5..df9ea429d 100644 --- a/internal/dump/tar.go +++ b/internal/dump/tar.go @@ -3,6 +3,7 @@ package dump import ( "archive/tar" "context" + "fmt" "os" "path/filepath" "strings" @@ -94,9 +95,8 @@ func (d *Dumper) dumpNodeTar(ctx context.Context, node *restic.Node, w *tar.Writ err = w.WriteHeader(header) if err != nil { - return errors.Wrap(err, "TarHeader") + return fmt.Errorf("writing header for %q: %w", node.Path, err) } - return d.writeNode(ctx, w, node) } diff --git a/internal/dump/tar_test.go b/internal/dump/tar_test.go index 0f2cb27a8..3556e6aeb 100644 --- a/internal/dump/tar_test.go +++ b/internal/dump/tar_test.go @@ -3,6 +3,8 @@ package dump import ( "archive/tar" "bytes" + "context" + "errors" "fmt" "io" "os" @@ -12,6 +14,8 @@ import ( "time" "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" ) func TestWriteTar(t *testing.T) { @@ -112,3 +116,29 @@ func checkTar(t *testing.T, testDir string, srcTar *bytes.Buffer) error { return nil } + +// #4307. +func TestFieldTooLong(t *testing.T) { + const maxSpecialFileSize = 1 << 20 // Unexported limit in archive/tar. + + node := restic.Node{ + Name: "file_with_xattr", + Path: "/file_with_xattr", + Type: "file", + Mode: 0644, + ExtendedAttributes: []restic.ExtendedAttribute{ + { + Name: "user.way_too_large", + Value: make([]byte, 2*maxSpecialFileSize), + }, + }, + } + + d := Dumper{format: "tar"} + err := d.dumpNodeTar(context.Background(), &node, tar.NewWriter(io.Discard)) + + // We want a tar.ErrFieldTooLong that has the filename. + rtest.Assert(t, errors.Is(err, tar.ErrFieldTooLong), "wrong type %T", err) + rtest.Assert(t, strings.Contains(err.Error(), node.Path), + "no filename in %q", err) +} From face5bd7f7a9b9bdfa652ba1cf85b43c52dd1aa2 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 23 Apr 2023 15:06:11 +0200 Subject: [PATCH 087/127] Document that the compression mode can be set via $RESTIC_COMPRESSION --- cmd/restic/global.go | 2 +- doc/manual_rest.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 3e6c5f3a9..5063f7d6b 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -126,7 +126,7 @@ func init() { f.StringVar(&globalOptions.TLSClientCertKeyFilename, "tls-client-cert", "", "path to a `file` containing PEM encoded TLS client certificate and private key") f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)") f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") - f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max)") + f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)") f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)") f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)") f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)") diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index 7f812f4e0..6bddd7ea3 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -50,7 +50,7 @@ Usage help is available: --cacert file file to load root certificates from (default: use system certificates) --cache-dir directory set the cache directory. (default: use system default cache directory) --cleanup-cache auto remove old cache directories - --compression mode compression mode (only available for repository format version 2), one of (auto|off|max) (default auto) + --compression mode compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) (default auto) -h, --help help for restic --insecure-tls skip TLS certificate verification when connecting to the repository (insecure) --json set output mode to JSON for commands that support it @@ -127,7 +127,7 @@ command: --cacert file file to load root certificates from (default: use system certificates) --cache-dir directory set the cache directory. (default: use system default cache directory) --cleanup-cache auto remove old cache directories - --compression mode compression mode (only available for repository format version 2), one of (auto|off|max) (default auto) + --compression mode compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) (default auto) --insecure-tls skip TLS certificate verification when connecting to the repository (insecure) --json set output mode to JSON for commands that support it --key-hint key key ID of key to try decrypting first (default: $RESTIC_KEY_HINT) From 37d0e323eb664e594a0c534350c82b00a656dcd0 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 30 Apr 2023 16:12:29 +0200 Subject: [PATCH 088/127] doc: update help output in manual_rest.rst --- doc/manual_rest.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index 6bddd7ea3..f812e3a70 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -26,7 +26,7 @@ Usage help is available: dump Print a backed-up file to stdout find Find a file, a directory or restic IDs forget Remove snapshots from the repository - generate Generate manual pages and auto-completion files (bash, fish, zsh) + generate Generate manual pages and auto-completion files (bash, fish, zsh, powershell) help Help about any command init Initialize a new repository key Manage keys (passwords) @@ -106,6 +106,7 @@ command: --files-from-raw file read the files to backup from file (can be combined with file args; can be specified multiple times) --files-from-verbatim file read the files to backup from file (can be combined with file args; can be specified multiple times) -f, --force force re-reading the target files/directories (overrides the "parent" flag) + -g, --group-by group group snapshots by host, paths and/or tags, separated by comma (disable grouping with '') (default host,paths) -h, --help help for backup -H, --host hostname set the hostname for the snapshot manually. To prevent an expensive rescan use the "parent" flag --iexclude pattern same as --exclude pattern but ignores the casing of filenames @@ -114,8 +115,8 @@ command: --ignore-inode ignore inode number changes when checking for modified files --no-scan do not run scanner to estimate size of backup -x, --one-file-system exclude other file systems, don't cross filesystem boundaries and subvolumes - --parent snapshot use this parent snapshot (default: last snapshot in the repository that has the same target files/directories, and is not newer than the snapshot time) - --read-concurrency n read n file concurrently (default: $RESTIC_READ_CONCURRENCY or 2) + --parent snapshot use this parent snapshot (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time) + --read-concurrency n read n files concurrently (default: $RESTIC_READ_CONCURRENCY or 2) --stdin read backup from stdin --stdin-filename filename filename to use when reading from stdin (default "stdin") --tag tags add tags for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times) (default []) From 7147a54cebfc6cdbbd98efc7e65f9f600dd2c7e9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 May 2023 01:59:10 +0000 Subject: [PATCH 089/127] build(deps): bump golang.org/x/oauth2 from 0.6.0 to 0.7.0 Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from 0.6.0 to 0.7.0. - [Release notes](https://github.com/golang/oauth2/releases) - [Commits](https://github.com/golang/oauth2/compare/v0.6.0...v0.7.0) --- updated-dependencies: - dependency-name: golang.org/x/oauth2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 752e88019..102554f85 100644 --- a/go.mod +++ b/go.mod @@ -25,12 +25,12 @@ require ( github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 golang.org/x/crypto v0.7.0 - golang.org/x/net v0.8.0 - golang.org/x/oauth2 v0.6.0 + golang.org/x/net v0.9.0 + golang.org/x/oauth2 v0.7.0 golang.org/x/sync v0.1.0 - golang.org/x/sys v0.6.0 - golang.org/x/term v0.6.0 - golang.org/x/text v0.8.0 + golang.org/x/sys v0.7.0 + golang.org/x/term v0.7.0 + golang.org/x/text v0.9.0 google.golang.org/api v0.116.0 ) diff --git a/go.sum b/go.sum index 493f95aad..d8d4462bb 100644 --- a/go.sum +++ b/go.sum @@ -189,11 +189,11 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= -golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -214,17 +214,17 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= From 6abd494915c78d5a7cbde28c3e6c4287770376ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 May 2023 01:59:50 +0000 Subject: [PATCH 090/127] build(deps): bump github.com/minio/minio-go/v7 from 7.0.50 to 7.0.52 Bumps [github.com/minio/minio-go/v7](https://github.com/minio/minio-go) from 7.0.50 to 7.0.52. - [Release notes](https://github.com/minio/minio-go/releases) - [Commits](https://github.com/minio/minio-go/compare/v7.0.50...v7.0.52) --- updated-dependencies: - dependency-name: github.com/minio/minio-go/v7 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 752e88019..e74048d65 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/juju/ratelimit v1.0.2 github.com/klauspost/compress v1.16.0 github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5 - github.com/minio/minio-go/v7 v7.0.50 + github.com/minio/minio-go/v7 v7.0.52 github.com/minio/sha256-simd v1.0.0 github.com/ncw/swift/v2 v2.0.1 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 493f95aad..f1ba771a3 100644 --- a/go.sum +++ b/go.sum @@ -116,8 +116,8 @@ github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5/go.mod h1:4FCXMUWo9 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.50 h1:4IL4V8m/kI90ZL6GupCARZVrBv8/XrcKcJhaJ3iz68k= -github.com/minio/minio-go/v7 v7.0.50/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU= +github.com/minio/minio-go/v7 v7.0.52 h1:8XhG36F6oKQUDDSuz6dY3rioMzovKjW40W6ANuN0Dps= +github.com/minio/minio-go/v7 v7.0.52/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= From 888c1ae63abd9c4886bdb3d33a23e12c7c34b760 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 May 2023 11:07:29 +0000 Subject: [PATCH 091/127] build(deps): bump github.com/klauspost/compress from 1.16.0 to 1.16.5 Bumps [github.com/klauspost/compress](https://github.com/klauspost/compress) from 1.16.0 to 1.16.5. - [Release notes](https://github.com/klauspost/compress/releases) - [Changelog](https://github.com/klauspost/compress/blob/master/.goreleaser.yml) - [Commits](https://github.com/klauspost/compress/compare/v1.16.0...v1.16.5) --- updated-dependencies: - dependency-name: github.com/klauspost/compress dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index aada19c36..c1d58d9d0 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/google/go-cmp v0.5.9 github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/juju/ratelimit v1.0.2 - github.com/klauspost/compress v1.16.0 + github.com/klauspost/compress v1.16.5 github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5 github.com/minio/minio-go/v7 v7.0.52 github.com/minio/sha256-simd v1.0.0 diff --git a/go.sum b/go.sum index 8b30b8ce4..953e72646 100644 --- a/go.sum +++ b/go.sum @@ -103,8 +103,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI= github.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= -github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= -github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= From d747a9c401ea7a320035b5f1f238b194c3524b7a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 May 2023 11:15:38 +0000 Subject: [PATCH 092/127] build(deps): bump golang.org/x/crypto from 0.7.0 to 0.8.0 Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.7.0 to 0.8.0. - [Release notes](https://github.com/golang/crypto/releases) - [Commits](https://github.com/golang/crypto/compare/v0.7.0...v0.8.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index aada19c36..386c55980 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/restic/chunker v0.4.0 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 - golang.org/x/crypto v0.7.0 + golang.org/x/crypto v0.8.0 golang.org/x/net v0.9.0 golang.org/x/oauth2 v0.7.0 golang.org/x/sync v0.1.0 diff --git a/go.sum b/go.sum index 8b30b8ce4..4cfa745f0 100644 --- a/go.sum +++ b/go.sum @@ -172,8 +172,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= From 9cef6b4c69d6151f50067b06a67dc383f5e79fec Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Thu, 6 Aug 2020 17:24:00 +0200 Subject: [PATCH 093/127] Add troubleshooting documentation --- doc/077_troubleshooting.rst | 105 ++++++++++++++++++++++++++++++++++++ doc/index.rst | 1 + 2 files changed, 106 insertions(+) create mode 100644 doc/077_troubleshooting.rst diff --git a/doc/077_troubleshooting.rst b/doc/077_troubleshooting.rst new file mode 100644 index 000000000..504853954 --- /dev/null +++ b/doc/077_troubleshooting.rst @@ -0,0 +1,105 @@ +.. + Normally, there are no heading levels assigned to certain characters as the structure is + determined from the succession of headings. However, this convention is used in Python’s + Style Guide for documenting which you may follow: + + # with overline, for parts + * for chapters + = for sections + - for subsections + ^ for subsubsections + " for paragraphs + +######################### +Troubleshooting +######################### + +Being a backup software, the repository format ensures that the data saved in the repository +is verifiable and error-restistant. Restic even implements some self-healing functionalities. + +However, situations might occur where your repository gets in an incorrect state and measurements +need to be done to get you out of this situation. These situations might be due to hardware failure, +accidentially removing files directly from the repository or bugs in the restic implementation. + +This document is meant to give you some hints about how to recover from such situations. + +1. Stay calm and don't over-react +******************************************** + +The most important thing if you find yourself in the situation of a damaged repository is to +stay calm and don't do anything you might regret later. + +The following point should be always considered: + +- Make a copy of you repository and try to recover from that copy. If you suspect a storage failure, + it may be even better, to make *two* copies: one to get all data out of the possibly failing storage + and another one to try the recovery process. +- Pause your regular operations on the repository or let them run on a copy. You will especially make + sure that no `forget` or `prune` is run as these command are supposed to remove data and may result + in data loss. +- Search if your issue is already known and solved. Good starting points are the restic forum and the + github issues. +- Get you some help if you are unsure what to do. Find a colleage or friend to discuss what should be done. + Also feel free to consult the restic forum. +- When using the commands below, make sure you read and understand the documentation. Some of the commands + may not be your every-day commands, so make sure you really understand what they are doing. + + +2. `check` is your friend +******************************************** + +Run `restic check` to find out what type of error you have. The results may be technical but can give you +a good hint what's really wrong. + +Moreover, you can always run a `check` to ensure that your repair really was sucessful and your repository +is in a sane state again. +But make sure that your needed data is also still contained in your repository ;-) + +Note that `check` also prints out warning in some cases. These warnings point out that the repo may be +optimized but is still in perfect shape and does not need any troubleshooting. + +3. Index trouble -> `rebuild-index` +******************************************** + +A common problem with broken repostories is that the index does no longer correctly represent the contents +of your pack files. This is especially the case if some pack files got lost. +`rebuild-index` recovers this situation and ensures that the index exactly represents the pack files. + +You might even need to manually remove corrupted pack files. In this case make sure, you run +`restic rebuild-index` after. + +Also if you encounter problems with the index files itselves, `rebuild-index` will solve these problems +immediately. + +However, rebuilding the index does not solve every problem, e.g. lost pack files. + +4. Delete unneeded defect snapshots -> `forget` +******************************************** + +If you encounter defect snapshots but realize you can spare them, it is often a good idea to simply +delete them using `forget`. In case that your repository remains with just sane snapshots (including +all trees and files) the next `prune` run will put your repository in a sane state. + +This can be also used if you manage to create new snapshots which can replace the defect ones, see +below. + +5. No fear to `backup` again +******************************************** + +There are quite some self-healing mechanisms withing the `backup` command. So it is always a good idea to +backup again and check if this did heal your repository. +If you realize that a specific file is broken in your repository and you have this file, any run of +`backup` which includes that file will be able to heal the situation. + +Note that `backup` relies on a correct index state, so make sure your index is fine or run `rebuild-index` +before running `backup`. + +6. Unreferenced tree -> `recover` +******************************************** + +If for some reason you have unreferenced trees in your repository but you actually need them, run +`recover` it will generate a new snapshot which allows access to all trees that you have in your +repository. + +Note that `recover` relies on a correct index state, so make sure your index is fine or run `rebuild-index` +before running `recover`. diff --git a/doc/index.rst b/doc/index.rst index 034dbda23..8b72dcf58 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -14,6 +14,7 @@ Restic Documentation 060_forget 070_encryption 075_scripting + 077_troubleshooting 080_examples 090_participating 100_references From 5f58797ba7d54c718caea3a14957151aa7f4a873 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Wed, 5 Aug 2020 21:32:15 +0200 Subject: [PATCH 094/127] Add repair command --- cmd/restic/cmd_repair.go | 269 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 269 insertions(+) create mode 100644 cmd/restic/cmd_repair.go diff --git a/cmd/restic/cmd_repair.go b/cmd/restic/cmd_repair.go new file mode 100644 index 000000000..aa3a9caca --- /dev/null +++ b/cmd/restic/cmd_repair.go @@ -0,0 +1,269 @@ +package main + +import ( + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/restic" + + "github.com/spf13/cobra" +) + +var cmdRepair = &cobra.Command{ + Use: "repair [flags] [snapshot ID] [...]", + Short: "Repair snapshots", + Long: ` +The "repair" command allows to repair broken snapshots. +It scans the given snapshots and generates new ones where +damaged tress and file contents are removed. +If the broken snapshots are deleted, a prune run will +be able to refit the repository. + +The command depends on a good state of the index, so if +there are inaccurancies in the index, make sure to run +"rebuild-index" before! + + +WARNING: +======== +Repairing and deleting broken snapshots causes data loss! +It will remove broken dirs and modify broken files in +the modified snapshots. + +If the contents of directories and files are still available, +the better option is to redo a backup which in that case is +able to "heal" already present snapshots. +Only use this command if you need to recover an old and +broken snapshot! + +EXIT STATUS +=========== + +Exit status is 0 if the command was successful, and non-zero if there was any error. +`, + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + return runRepair(repairOptions, args) + }, +} + +// RestoreOptions collects all options for the restore command. +type RepairOptions struct { + Hosts []string + Paths []string + Tags restic.TagLists + AddTag string + Append string + DryRun bool + DeleteSnapshots bool +} + +var repairOptions RepairOptions + +func init() { + cmdRoot.AddCommand(cmdRepair) + flags := cmdRepair.Flags() + flags.StringArrayVarP(&repairOptions.Hosts, "host", "H", nil, `only consider snapshots for this host (can be specified multiple times)`) + flags.Var(&repairOptions.Tags, "tag", "only consider snapshots which include this `taglist`") + flags.StringArrayVar(&repairOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`") + flags.StringVar(&repairOptions.AddTag, "add-tag", "repaired", "tag to add to repaired snapshots") + flags.StringVar(&repairOptions.Append, "append", ".repaired", "string to append to repaired dirs/files; remove files if emtpy or impossible to repair") + flags.BoolVarP(&repairOptions.DryRun, "dry-run", "n", true, "don't do anything, only show what would be done") + flags.BoolVar(&repairOptions.DeleteSnapshots, "delete-snapshots", false, "delete original snapshots") +} + +func runRepair(opts RepairOptions, args []string) error { + switch { + case opts.DryRun: + Printf("\n note: --dry-run is set\n-> repair will only show what it would do.\n\n") + case opts.DeleteSnapshots: + Printf("\n note: --dry-run is not set and --delete is set\n-> this may result in data loss!\n\n") + } + + repo, err := OpenRepository(globalOptions) + if err != nil { + return err + } + + lock, err := lockRepoExclusive(globalOptions.ctx, repo) + defer unlockRepo(lock) + if err != nil { + return err + } + + if err := repo.LoadIndex(globalOptions.ctx); err != nil { + return err + } + + // get snapshots to check & repair + var snapshots []*restic.Snapshot + for sn := range FindFilteredSnapshots(globalOptions.ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { + snapshots = append(snapshots, sn) + } + + return repairSnapshots(opts, repo, snapshots) +} + +func repairSnapshots(opts RepairOptions, repo restic.Repository, snapshots []*restic.Snapshot) error { + ctx := globalOptions.ctx + + replaces := make(idMap) + seen := restic.NewIDSet() + deleteSn := restic.NewIDSet() + + Verbosef("check and repair %d snapshots\n", len(snapshots)) + bar := newProgressMax(!globalOptions.Quiet, uint64(len(snapshots)), "snapshots") + for _, sn := range snapshots { + debug.Log("process snapshot %v", sn.ID()) + Printf("%v:\n", sn) + newID, changed, err := repairTree(opts, repo, "/", *sn.Tree, replaces, seen) + switch { + case err != nil: + Printf("the root tree is damaged -> delete snapshot.\n") + deleteSn.Insert(*sn.ID()) + case changed: + err = changeSnapshot(opts, repo, sn, newID) + if err != nil { + return err + } + deleteSn.Insert(*sn.ID()) + default: + Printf("is ok.\n") + } + debug.Log("processed snapshot %v", sn.ID()) + bar.Add(1) + } + bar.Done() + + err := repo.Flush(ctx) + if err != nil { + return err + } + + if len(deleteSn) > 0 && opts.DeleteSnapshots { + Verbosef("delete %d snapshots...\n", len(deleteSn)) + if !opts.DryRun { + DeleteFiles(globalOptions, repo, deleteSn, restic.SnapshotFile) + } + } + return nil +} + +// changeSnapshot creates a modified snapshot: +// - set the tree to newID +// - add the rag opts.AddTag +// - preserve original ID +// if opts.DryRun is set, it doesn't change anything but only +func changeSnapshot(opts RepairOptions, repo restic.Repository, sn *restic.Snapshot, newID restic.ID) error { + sn.AddTags([]string{opts.AddTag}) + // Retain the original snapshot id over all tag changes. + if sn.Original == nil { + sn.Original = sn.ID() + } + sn.Tree = &newID + if !opts.DryRun { + newID, err := repo.SaveJSONUnpacked(globalOptions.ctx, restic.SnapshotFile, sn) + if err != nil { + return err + } + Printf("snapshot repaired -> %v created.\n", newID.Str()) + } else { + Printf("would have repaired snpshot %v.\n", sn.ID().Str()) + } + return nil +} + +type idMap map[restic.ID]restic.ID + +// repairTree checks and repairs a tree and all its subtrees +// Two error cases are checked: +// - trees which cannot be loaded (-> the tree contents will be removed) +// - files whose contents are not fully available (-> file will be modified) +// In case of an error, the changes made depends on: +// - opts.Append: string to append to "repared" names; if empty files will not repaired but deleted +// - opts.DryRun: if set to true, only print out what to but don't change anything +func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID restic.ID, replaces idMap, seen restic.IDSet) (restic.ID, bool, error) { + ctx := globalOptions.ctx + + // check if tree was already changed + newID, ok := replaces[treeID] + if ok { + return newID, true, nil + } + + // check if tree was seen but not changed + if seen.Has(treeID) { + return treeID, false, nil + } + + tree, err := repo.LoadTree(ctx, treeID) + if err != nil { + return newID, false, err + } + + var newNodes []*restic.Node + changed := false + for _, node := range tree.Nodes { + switch node.Type { + case "file": + ok := true + var newContent restic.IDs + var newSize uint64 + // check all contents and remove if not available + for _, id := range node.Content { + if size, found := repo.LookupBlobSize(id, restic.DataBlob); !found { + ok = false + } else { + newContent = append(newContent, id) + newSize += uint64(size) + } + } + if !ok { + changed = true + if opts.Append == "" || newSize == 0 { + Printf("removed defect file '%v'\n", path+node.Name) + continue + } + Printf("repaired defect file '%v'", path+node.Name) + node.Name = node.Name + opts.Append + Printf(" to '%v'\n", node.Name) + node.Content = newContent + node.Size = newSize + } + case "dir": + // rewrite if necessary + newID, c, err := repairTree(opts, repo, path+node.Name+"/", *node.Subtree, replaces, seen) + switch { + case err != nil: + // If we get an error, we remove this subtree + changed = true + Printf("removed defect dir '%v'", path+node.Name) + node.Name = node.Name + opts.Append + Printf("(now emtpy '%v')\n", node.Name) + node.Subtree = nil + case c: + node.Subtree = &newID + changed = true + } + } + newNodes = append(newNodes, node) + } + + if !changed { + seen.Insert(treeID) + return treeID, false, nil + } + + tree.Nodes = newNodes + + if !opts.DryRun { + newID, err = repo.SaveTree(ctx, tree) + if err != nil { + return newID, false, err + } + Printf("modified tree %v, new id: %v\n", treeID.Str(), newID.Str()) + } else { + Printf("would have modified tree %v\n", treeID.Str()) + } + + replaces[treeID] = newID + return newID, true, nil +} From 6557f36f615c58bf98772bf6ee49b289d9670ef1 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Thu, 6 Aug 2020 12:31:37 +0200 Subject: [PATCH 095/127] Add changelog and docu for #2876 --- changelog/unreleased/issue-1759 | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 changelog/unreleased/issue-1759 diff --git a/changelog/unreleased/issue-1759 b/changelog/unreleased/issue-1759 new file mode 100644 index 000000000..0b0d28aa5 --- /dev/null +++ b/changelog/unreleased/issue-1759 @@ -0,0 +1,16 @@ +Enhancement: Add new command repair + +We've added a new command repair which allows to repair snapshots even if needed +parts of it are not accessable in the repository. Note that using this command +can lead to data loss! + +Some corrupted repositories were reported in several issues and so far restic +lacked a possibility to accept data loss but clean those up such that the +repository returns to a sane state. This possibility was now added. + +https://github.com/restic/restic/issues/1759 +https://github.com/restic/restic/issues/1798 +https://github.com/restic/restic/issues/2334 +https://github.com/restic/restic/pull/2876 +https://forum.restic.net/t/corrupted-repo-how-to-repair/799 +https://forum.restic.net/t/recovery-options-for-damaged-repositories/1571 From 99a05d5ab23f16a6d6eed08d0315deea563e3f50 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Thu, 6 Aug 2020 17:24:00 +0200 Subject: [PATCH 096/127] Update troubleshooting documentation --- doc/077_troubleshooting.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/077_troubleshooting.rst b/doc/077_troubleshooting.rst index 504853954..50c19565e 100644 --- a/doc/077_troubleshooting.rst +++ b/doc/077_troubleshooting.rst @@ -103,3 +103,9 @@ repository. Note that `recover` relies on a correct index state, so make sure your index is fine or run `rebuild-index` before running `recover`. + +7. Repair defect snapshots using `repair` +******************************************** + +If all other things did not help, you can repair defect snapshots with `repair`. Note that the repaired +snapshots will miss data which was referenced in the defect snapshot. From 08ae708b3b2eaa2c9f6b614737b2589adc1a562d Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Sun, 22 Nov 2020 22:33:02 +0100 Subject: [PATCH 097/127] make linter happy --- cmd/restic/cmd_repair.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/restic/cmd_repair.go b/cmd/restic/cmd_repair.go index aa3a9caca..e159da261 100644 --- a/cmd/restic/cmd_repair.go +++ b/cmd/restic/cmd_repair.go @@ -84,7 +84,8 @@ func runRepair(opts RepairOptions, args []string) error { } lock, err := lockRepoExclusive(globalOptions.ctx, repo) - defer unlockRepo(lock) + // to make linter happy, as unlockRepo returns an error (which is ignored) + defer func() { _ = unlockRepo(lock) }() if err != nil { return err } From d23a2e192576da40f8da86ae31182fab8115b189 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Sat, 20 Feb 2021 20:16:05 +0100 Subject: [PATCH 098/127] better error handling and correct nil tree behavior --- cmd/restic/cmd_repair.go | 72 ++++++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 21 deletions(-) diff --git a/cmd/restic/cmd_repair.go b/cmd/restic/cmd_repair.go index e159da261..47beaf3cf 100644 --- a/cmd/restic/cmd_repair.go +++ b/cmd/restic/cmd_repair.go @@ -84,8 +84,7 @@ func runRepair(opts RepairOptions, args []string) error { } lock, err := lockRepoExclusive(globalOptions.ctx, repo) - // to make linter happy, as unlockRepo returns an error (which is ignored) - defer func() { _ = unlockRepo(lock) }() + defer unlockRepo(lock) if err != nil { return err } @@ -115,9 +114,11 @@ func repairSnapshots(opts RepairOptions, repo restic.Repository, snapshots []*re for _, sn := range snapshots { debug.Log("process snapshot %v", sn.ID()) Printf("%v:\n", sn) - newID, changed, err := repairTree(opts, repo, "/", *sn.Tree, replaces, seen) + newID, changed, lErr, err := repairTree(opts, repo, "/", sn.Tree, replaces, seen) switch { case err != nil: + return err + case lErr: Printf("the root tree is damaged -> delete snapshot.\n") deleteSn.Insert(*sn.ID()) case changed: @@ -153,13 +154,13 @@ func repairSnapshots(opts RepairOptions, repo restic.Repository, snapshots []*re // - add the rag opts.AddTag // - preserve original ID // if opts.DryRun is set, it doesn't change anything but only -func changeSnapshot(opts RepairOptions, repo restic.Repository, sn *restic.Snapshot, newID restic.ID) error { +func changeSnapshot(opts RepairOptions, repo restic.Repository, sn *restic.Snapshot, newID *restic.ID) error { sn.AddTags([]string{opts.AddTag}) // Retain the original snapshot id over all tag changes. if sn.Original == nil { sn.Original = sn.ID() } - sn.Tree = &newID + sn.Tree = newID if !opts.DryRun { newID, err := repo.SaveJSONUnpacked(globalOptions.ctx, restic.SnapshotFile, sn) if err != nil { @@ -175,29 +176,43 @@ func changeSnapshot(opts RepairOptions, repo restic.Repository, sn *restic.Snaps type idMap map[restic.ID]restic.ID // repairTree checks and repairs a tree and all its subtrees -// Two error cases are checked: +// Three error cases are checked: +// - tree is a nil tree (-> will be replaced by an empty tree) // - trees which cannot be loaded (-> the tree contents will be removed) // - files whose contents are not fully available (-> file will be modified) // In case of an error, the changes made depends on: // - opts.Append: string to append to "repared" names; if empty files will not repaired but deleted // - opts.DryRun: if set to true, only print out what to but don't change anything -func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID restic.ID, replaces idMap, seen restic.IDSet) (restic.ID, bool, error) { +// Returns: +// - the new ID +// - whether the ID changed +// - whether there was a load error when loading this tre +// - error for other errors (these are errors when saving a tree) +func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID *restic.ID, replaces idMap, seen restic.IDSet) (*restic.ID, bool, bool, error) { ctx := globalOptions.ctx + // handle and repair nil trees + if treeID == nil { + empty, err := emptyTree(opts.DryRun, repo) + Printf("repaired nil tree '%v'\n", path) + return &empty, true, false, err + } + // check if tree was already changed - newID, ok := replaces[treeID] + newID, ok := replaces[*treeID] if ok { - return newID, true, nil + return &newID, true, false, nil } // check if tree was seen but not changed - if seen.Has(treeID) { - return treeID, false, nil + if seen.Has(*treeID) { + return treeID, false, false, nil } - tree, err := repo.LoadTree(ctx, treeID) + tree, err := repo.LoadTree(ctx, *treeID) if err != nil { - return newID, false, err + // mark as load error + return &newID, false, true, nil } var newNodes []*restic.Node @@ -231,17 +246,23 @@ func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID } case "dir": // rewrite if necessary - newID, c, err := repairTree(opts, repo, path+node.Name+"/", *node.Subtree, replaces, seen) + newID, c, lErr, err := repairTree(opts, repo, path+node.Name+"/", node.Subtree, replaces, seen) switch { case err != nil: + return newID, true, false, err + case lErr: // If we get an error, we remove this subtree changed = true Printf("removed defect dir '%v'", path+node.Name) node.Name = node.Name + opts.Append Printf("(now emtpy '%v')\n", node.Name) - node.Subtree = nil + empty, err := emptyTree(opts.DryRun, repo) + if err != nil { + return newID, true, false, err + } + node.Subtree = &empty case c: - node.Subtree = &newID + node.Subtree = newID changed = true } } @@ -249,8 +270,8 @@ func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID } if !changed { - seen.Insert(treeID) - return treeID, false, nil + seen.Insert(*treeID) + return treeID, false, false, nil } tree.Nodes = newNodes @@ -258,13 +279,22 @@ func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID if !opts.DryRun { newID, err = repo.SaveTree(ctx, tree) if err != nil { - return newID, false, err + return &newID, true, false, err } Printf("modified tree %v, new id: %v\n", treeID.Str(), newID.Str()) } else { Printf("would have modified tree %v\n", treeID.Str()) } - replaces[treeID] = newID - return newID, true, nil + replaces[*treeID] = newID + return &newID, true, false, nil +} + +func emptyTree(dryRun bool, repo restic.Repository) (restic.ID, error) { + ctx := globalOptions.ctx + var tree restic.Tree + if !dryRun { + return repo.SaveTree(ctx, &tree) + } + return restic.ID{}, nil } From 947f0c345e3f298f4971e6acedbea9eec28e2ac0 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Sat, 20 Feb 2021 20:56:03 +0100 Subject: [PATCH 099/127] correct typos --- cmd/restic/cmd_repair.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/restic/cmd_repair.go b/cmd/restic/cmd_repair.go index 47beaf3cf..19574c43d 100644 --- a/cmd/restic/cmd_repair.go +++ b/cmd/restic/cmd_repair.go @@ -45,7 +45,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er }, } -// RestoreOptions collects all options for the restore command. +// RepairOptions collects all options for the repair command. type RepairOptions struct { Hosts []string Paths []string @@ -65,7 +65,7 @@ func init() { flags.Var(&repairOptions.Tags, "tag", "only consider snapshots which include this `taglist`") flags.StringArrayVar(&repairOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`") flags.StringVar(&repairOptions.AddTag, "add-tag", "repaired", "tag to add to repaired snapshots") - flags.StringVar(&repairOptions.Append, "append", ".repaired", "string to append to repaired dirs/files; remove files if emtpy or impossible to repair") + flags.StringVar(&repairOptions.Append, "append", ".repaired", "string to append to repaired dirs/files; remove files if empty or impossible to repair") flags.BoolVarP(&repairOptions.DryRun, "dry-run", "n", true, "don't do anything, only show what would be done") flags.BoolVar(&repairOptions.DeleteSnapshots, "delete-snapshots", false, "delete original snapshots") } @@ -75,7 +75,7 @@ func runRepair(opts RepairOptions, args []string) error { case opts.DryRun: Printf("\n note: --dry-run is set\n-> repair will only show what it would do.\n\n") case opts.DeleteSnapshots: - Printf("\n note: --dry-run is not set and --delete is set\n-> this may result in data loss!\n\n") + Printf("\n note: --dry-run is not set and --delete-snapshots is set\n-> this may result in data loss!\n\n") } repo, err := OpenRepository(globalOptions) @@ -168,7 +168,7 @@ func changeSnapshot(opts RepairOptions, repo restic.Repository, sn *restic.Snaps } Printf("snapshot repaired -> %v created.\n", newID.Str()) } else { - Printf("would have repaired snpshot %v.\n", sn.ID().Str()) + Printf("would have repaired snapshot %v.\n", sn.ID().Str()) } return nil } @@ -235,10 +235,10 @@ func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID if !ok { changed = true if opts.Append == "" || newSize == 0 { - Printf("removed defect file '%v'\n", path+node.Name) + Printf("removed defective file '%v'\n", path+node.Name) continue } - Printf("repaired defect file '%v'", path+node.Name) + Printf("repaired defective file '%v'", path+node.Name) node.Name = node.Name + opts.Append Printf(" to '%v'\n", node.Name) node.Content = newContent @@ -253,9 +253,9 @@ func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID case lErr: // If we get an error, we remove this subtree changed = true - Printf("removed defect dir '%v'", path+node.Name) + Printf("removed defective dir '%v'", path+node.Name) node.Name = node.Name + opts.Append - Printf("(now emtpy '%v')\n", node.Name) + Printf("(now empty '%v')\n", node.Name) empty, err := emptyTree(opts.DryRun, repo) if err != nil { return newID, true, false, err From a14a63cd29204908c5a9a37a5cca93a1cea12b7c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 10 Dec 2022 17:18:04 +0100 Subject: [PATCH 100/127] modernize code --- cmd/restic/cmd_repair.go | 105 ++++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 52 deletions(-) diff --git a/cmd/restic/cmd_repair.go b/cmd/restic/cmd_repair.go index 19574c43d..30fe8be1c 100644 --- a/cmd/restic/cmd_repair.go +++ b/cmd/restic/cmd_repair.go @@ -1,8 +1,11 @@ package main import ( + "context" + "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" + "golang.org/x/sync/errgroup" "github.com/spf13/cobra" ) @@ -41,15 +44,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runRepair(repairOptions, args) + return runRepair(cmd.Context(), globalOptions, repairOptions, args) }, } // RepairOptions collects all options for the repair command. type RepairOptions struct { - Hosts []string - Paths []string - Tags restic.TagLists + restic.SnapshotFilter + AddTag string Append string DryRun bool @@ -61,16 +63,16 @@ var repairOptions RepairOptions func init() { cmdRoot.AddCommand(cmdRepair) flags := cmdRepair.Flags() - flags.StringArrayVarP(&repairOptions.Hosts, "host", "H", nil, `only consider snapshots for this host (can be specified multiple times)`) - flags.Var(&repairOptions.Tags, "tag", "only consider snapshots which include this `taglist`") - flags.StringArrayVar(&repairOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`") + + initMultiSnapshotFilter(flags, &repairOptions.SnapshotFilter, true) + flags.StringVar(&repairOptions.AddTag, "add-tag", "repaired", "tag to add to repaired snapshots") flags.StringVar(&repairOptions.Append, "append", ".repaired", "string to append to repaired dirs/files; remove files if empty or impossible to repair") flags.BoolVarP(&repairOptions.DryRun, "dry-run", "n", true, "don't do anything, only show what would be done") flags.BoolVar(&repairOptions.DeleteSnapshots, "delete-snapshots", false, "delete original snapshots") } -func runRepair(opts RepairOptions, args []string) error { +func runRepair(ctx context.Context, gopts GlobalOptions, opts RepairOptions, args []string) error { switch { case opts.DryRun: Printf("\n note: --dry-run is set\n-> repair will only show what it would do.\n\n") @@ -78,64 +80,67 @@ func runRepair(opts RepairOptions, args []string) error { Printf("\n note: --dry-run is not set and --delete-snapshots is set\n-> this may result in data loss!\n\n") } - repo, err := OpenRepository(globalOptions) + repo, err := OpenRepository(ctx, globalOptions) if err != nil { return err } - lock, err := lockRepoExclusive(globalOptions.ctx, repo) + lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err } - if err := repo.LoadIndex(globalOptions.ctx); err != nil { + if err := repo.LoadIndex(ctx); err != nil { return err } // get snapshots to check & repair var snapshots []*restic.Snapshot - for sn := range FindFilteredSnapshots(globalOptions.ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { + for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) { snapshots = append(snapshots, sn) } - return repairSnapshots(opts, repo, snapshots) + return repairSnapshots(ctx, opts, repo, snapshots) } -func repairSnapshots(opts RepairOptions, repo restic.Repository, snapshots []*restic.Snapshot) error { - ctx := globalOptions.ctx - +func repairSnapshots(ctx context.Context, opts RepairOptions, repo restic.Repository, snapshots []*restic.Snapshot) error { replaces := make(idMap) seen := restic.NewIDSet() deleteSn := restic.NewIDSet() Verbosef("check and repair %d snapshots\n", len(snapshots)) bar := newProgressMax(!globalOptions.Quiet, uint64(len(snapshots)), "snapshots") - for _, sn := range snapshots { - debug.Log("process snapshot %v", sn.ID()) - Printf("%v:\n", sn) - newID, changed, lErr, err := repairTree(opts, repo, "/", sn.Tree, replaces, seen) - switch { - case err != nil: - return err - case lErr: - Printf("the root tree is damaged -> delete snapshot.\n") - deleteSn.Insert(*sn.ID()) - case changed: - err = changeSnapshot(opts, repo, sn, newID) - if err != nil { + wg, ctx := errgroup.WithContext(ctx) + repo.StartPackUploader(ctx, wg) + wg.Go(func() error { + for _, sn := range snapshots { + debug.Log("process snapshot %v", sn.ID()) + Printf("%v:\n", sn) + newID, changed, lErr, err := repairTree(ctx, opts, repo, "/", sn.Tree, replaces, seen) + switch { + case err != nil: return err + case lErr: + Printf("the root tree is damaged -> delete snapshot.\n") + deleteSn.Insert(*sn.ID()) + case changed: + err = changeSnapshot(ctx, opts, repo, sn, newID) + if err != nil { + return err + } + deleteSn.Insert(*sn.ID()) + default: + Printf("is ok.\n") } - deleteSn.Insert(*sn.ID()) - default: - Printf("is ok.\n") + debug.Log("processed snapshot %v", sn.ID()) + bar.Add(1) } - debug.Log("processed snapshot %v", sn.ID()) - bar.Add(1) - } - bar.Done() + bar.Done() + return repo.Flush(ctx) + }) - err := repo.Flush(ctx) + err := wg.Wait() if err != nil { return err } @@ -143,7 +148,7 @@ func repairSnapshots(opts RepairOptions, repo restic.Repository, snapshots []*re if len(deleteSn) > 0 && opts.DeleteSnapshots { Verbosef("delete %d snapshots...\n", len(deleteSn)) if !opts.DryRun { - DeleteFiles(globalOptions, repo, deleteSn, restic.SnapshotFile) + DeleteFiles(ctx, globalOptions, repo, deleteSn, restic.SnapshotFile) } } return nil @@ -154,7 +159,7 @@ func repairSnapshots(opts RepairOptions, repo restic.Repository, snapshots []*re // - add the rag opts.AddTag // - preserve original ID // if opts.DryRun is set, it doesn't change anything but only -func changeSnapshot(opts RepairOptions, repo restic.Repository, sn *restic.Snapshot, newID *restic.ID) error { +func changeSnapshot(ctx context.Context, opts RepairOptions, repo restic.Repository, sn *restic.Snapshot, newID *restic.ID) error { sn.AddTags([]string{opts.AddTag}) // Retain the original snapshot id over all tag changes. if sn.Original == nil { @@ -162,7 +167,7 @@ func changeSnapshot(opts RepairOptions, repo restic.Repository, sn *restic.Snaps } sn.Tree = newID if !opts.DryRun { - newID, err := repo.SaveJSONUnpacked(globalOptions.ctx, restic.SnapshotFile, sn) + newID, err := restic.SaveSnapshot(ctx, repo, sn) if err != nil { return err } @@ -188,12 +193,10 @@ type idMap map[restic.ID]restic.ID // - whether the ID changed // - whether there was a load error when loading this tre // - error for other errors (these are errors when saving a tree) -func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID *restic.ID, replaces idMap, seen restic.IDSet) (*restic.ID, bool, bool, error) { - ctx := globalOptions.ctx - +func repairTree(ctx context.Context, opts RepairOptions, repo restic.Repository, path string, treeID *restic.ID, replaces idMap, seen restic.IDSet) (*restic.ID, bool, bool, error) { // handle and repair nil trees if treeID == nil { - empty, err := emptyTree(opts.DryRun, repo) + empty, err := emptyTree(ctx, repo, opts.DryRun) Printf("repaired nil tree '%v'\n", path) return &empty, true, false, err } @@ -209,7 +212,7 @@ func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID return treeID, false, false, nil } - tree, err := repo.LoadTree(ctx, *treeID) + tree, err := restic.LoadTree(ctx, repo, *treeID) if err != nil { // mark as load error return &newID, false, true, nil @@ -246,7 +249,7 @@ func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID } case "dir": // rewrite if necessary - newID, c, lErr, err := repairTree(opts, repo, path+node.Name+"/", node.Subtree, replaces, seen) + newID, c, lErr, err := repairTree(ctx, opts, repo, path+node.Name+"/", node.Subtree, replaces, seen) switch { case err != nil: return newID, true, false, err @@ -256,7 +259,7 @@ func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID Printf("removed defective dir '%v'", path+node.Name) node.Name = node.Name + opts.Append Printf("(now empty '%v')\n", node.Name) - empty, err := emptyTree(opts.DryRun, repo) + empty, err := emptyTree(ctx, repo, opts.DryRun) if err != nil { return newID, true, false, err } @@ -277,7 +280,7 @@ func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID tree.Nodes = newNodes if !opts.DryRun { - newID, err = repo.SaveTree(ctx, tree) + newID, err = restic.SaveTree(ctx, repo, tree) if err != nil { return &newID, true, false, err } @@ -290,11 +293,9 @@ func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID return &newID, true, false, nil } -func emptyTree(dryRun bool, repo restic.Repository) (restic.ID, error) { - ctx := globalOptions.ctx - var tree restic.Tree +func emptyTree(ctx context.Context, repo restic.Repository, dryRun bool) (restic.ID, error) { if !dryRun { - return repo.SaveTree(ctx, &tree) + return restic.SaveTree(ctx, repo, &restic.Tree{}) } return restic.ID{}, nil } From db459eda21016aa4f60c2f14c0babd11f5a0f8c4 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 10 Dec 2022 17:25:38 +0100 Subject: [PATCH 101/127] move to subcommand --- ...{cmd_repair.go => cmd_repair_snapshots.go} | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) rename cmd/restic/{cmd_repair.go => cmd_repair_snapshots.go} (87%) diff --git a/cmd/restic/cmd_repair.go b/cmd/restic/cmd_repair_snapshots.go similarity index 87% rename from cmd/restic/cmd_repair.go rename to cmd/restic/cmd_repair_snapshots.go index 30fe8be1c..a1e6b7f61 100644 --- a/cmd/restic/cmd_repair.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -11,10 +11,15 @@ import ( ) var cmdRepair = &cobra.Command{ - Use: "repair [flags] [snapshot ID] [...]", + Use: "repair", + Short: "Repair commands", +} + +var cmdRepairSnapshots = &cobra.Command{ + Use: "snapshots [flags] [snapshot ID] [...]", Short: "Repair snapshots", Long: ` -The "repair" command allows to repair broken snapshots. +The "repair snapshots" command allows to repair broken snapshots. It scans the given snapshots and generates new ones where damaged tress and file contents are removed. If the broken snapshots are deleted, a prune run will @@ -44,7 +49,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runRepair(cmd.Context(), globalOptions, repairOptions, args) + return runRepairSnapshots(cmd.Context(), globalOptions, repairSnapshotOptions, args) }, } @@ -58,21 +63,22 @@ type RepairOptions struct { DeleteSnapshots bool } -var repairOptions RepairOptions +var repairSnapshotOptions RepairOptions func init() { cmdRoot.AddCommand(cmdRepair) - flags := cmdRepair.Flags() + cmdRepair.AddCommand(cmdRepairSnapshots) + flags := cmdRepairSnapshots.Flags() - initMultiSnapshotFilter(flags, &repairOptions.SnapshotFilter, true) + initMultiSnapshotFilter(flags, &repairSnapshotOptions.SnapshotFilter, true) - flags.StringVar(&repairOptions.AddTag, "add-tag", "repaired", "tag to add to repaired snapshots") - flags.StringVar(&repairOptions.Append, "append", ".repaired", "string to append to repaired dirs/files; remove files if empty or impossible to repair") - flags.BoolVarP(&repairOptions.DryRun, "dry-run", "n", true, "don't do anything, only show what would be done") - flags.BoolVar(&repairOptions.DeleteSnapshots, "delete-snapshots", false, "delete original snapshots") + flags.StringVar(&repairSnapshotOptions.AddTag, "add-tag", "repaired", "tag to add to repaired snapshots") + flags.StringVar(&repairSnapshotOptions.Append, "append", ".repaired", "string to append to repaired dirs/files; remove files if empty or impossible to repair") + flags.BoolVarP(&repairSnapshotOptions.DryRun, "dry-run", "n", true, "don't do anything, only show what would be done") + flags.BoolVar(&repairSnapshotOptions.DeleteSnapshots, "delete-snapshots", false, "delete original snapshots") } -func runRepair(ctx context.Context, gopts GlobalOptions, opts RepairOptions, args []string) error { +func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOptions, args []string) error { switch { case opts.DryRun: Printf("\n note: --dry-run is set\n-> repair will only show what it would do.\n\n") From 118d599d0ac0053560262e17af3dda8eb92d770a Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Tue, 27 Dec 2022 18:25:39 +0100 Subject: [PATCH 102/127] Rename 'rebuild-index' to 'repair index' The old name still works, but is deprecated. --- cmd/restic/cmd_check.go | 2 +- cmd/restic/cmd_prune.go | 2 +- cmd/restic/cmd_repair.go | 14 ++++++++ ...d_rebuild_index.go => cmd_repair_index.go} | 36 ++++++++++++------- cmd/restic/cmd_repair_snapshots.go | 9 +---- cmd/restic/integration_test.go | 14 ++++---- doc/060_forget.rst | 2 +- doc/077_troubleshooting.rst | 12 +++---- doc/manual_rest.rst | 2 +- internal/archiver/archiver.go | 2 +- 10 files changed, 57 insertions(+), 38 deletions(-) create mode 100644 cmd/restic/cmd_repair.go rename cmd/restic/{cmd_rebuild_index.go => cmd_repair_index.go} (75%) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index e5f29a7e5..b9f3199b2 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -245,7 +245,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args } if suggestIndexRebuild { - Printf("Duplicate packs/old indexes are non-critical, you can run `restic rebuild-index' to correct this.\n") + Printf("Duplicate packs/old indexes are non-critical, you can run `restic repair index' to correct this.\n") } if mixedFound { Printf("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n") diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index 6104002b0..1138bb55b 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -488,7 +488,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Reposi // Pack size does not fit and pack is needed => error // If the pack is not needed, this is no error, the pack can // and will be simply removed, see below. - Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic rebuild-index'.\n", + Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic repair index'.\n", id.Str(), p.unusedSize+p.usedSize, packSize) return errorSizeNotMatching } diff --git a/cmd/restic/cmd_repair.go b/cmd/restic/cmd_repair.go new file mode 100644 index 000000000..aefe02f3c --- /dev/null +++ b/cmd/restic/cmd_repair.go @@ -0,0 +1,14 @@ +package main + +import ( + "github.com/spf13/cobra" +) + +var cmdRepair = &cobra.Command{ + Use: "repair", + Short: "Repair the repository", +} + +func init() { + cmdRoot.AddCommand(cmdRepair) +} diff --git a/cmd/restic/cmd_rebuild_index.go b/cmd/restic/cmd_repair_index.go similarity index 75% rename from cmd/restic/cmd_rebuild_index.go rename to cmd/restic/cmd_repair_index.go index 5d70a9e12..25d6b1cab 100644 --- a/cmd/restic/cmd_rebuild_index.go +++ b/cmd/restic/cmd_repair_index.go @@ -7,15 +7,15 @@ import ( "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" - "github.com/spf13/cobra" + "github.com/spf13/pflag" ) -var cmdRebuildIndex = &cobra.Command{ - Use: "rebuild-index [flags]", +var cmdRepairIndex = &cobra.Command{ + Use: "index [flags]", Short: "Build a new index", Long: ` -The "rebuild-index" command creates a new index based on the pack files in the +The "repair index" command creates a new index based on the pack files in the repository. EXIT STATUS @@ -25,25 +25,37 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runRebuildIndex(cmd.Context(), rebuildIndexOptions, globalOptions) + return runRebuildIndex(cmd.Context(), repairIndexOptions, globalOptions) }, } -// RebuildIndexOptions collects all options for the rebuild-index command. -type RebuildIndexOptions struct { +var cmdRebuildIndex = &cobra.Command{ + Use: "rebuild-index [flags]", + Short: cmdRepairIndex.Short, + Long: cmdRepairIndex.Long, + Deprecated: `Use "repair index" instead`, + DisableAutoGenTag: true, + RunE: cmdRepairIndex.RunE, +} + +// RepairIndexOptions collects all options for the repair index command. +type RepairIndexOptions struct { ReadAllPacks bool } -var rebuildIndexOptions RebuildIndexOptions +var repairIndexOptions RepairIndexOptions func init() { + cmdRepair.AddCommand(cmdRepairIndex) + // add alias for old name cmdRoot.AddCommand(cmdRebuildIndex) - f := cmdRebuildIndex.Flags() - f.BoolVar(&rebuildIndexOptions.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch") + for _, f := range []*pflag.FlagSet{cmdRepairIndex.Flags(), cmdRebuildIndex.Flags()} { + f.BoolVar(&repairIndexOptions.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch") + } } -func runRebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions) error { +func runRebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions) error { repo, err := OpenRepository(ctx, gopts) if err != nil { return err @@ -58,7 +70,7 @@ func runRebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts Global return rebuildIndex(ctx, opts, gopts, repo, restic.NewIDSet()) } -func rebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error { +func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error { var obsoleteIndexes restic.IDs packSizeFromList := make(map[restic.ID]int64) packSizeFromIndex := make(map[restic.ID]int64) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index a1e6b7f61..8b9005900 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -10,11 +10,6 @@ import ( "github.com/spf13/cobra" ) -var cmdRepair = &cobra.Command{ - Use: "repair", - Short: "Repair commands", -} - var cmdRepairSnapshots = &cobra.Command{ Use: "snapshots [flags] [snapshot ID] [...]", Short: "Repair snapshots", @@ -27,7 +22,7 @@ be able to refit the repository. The command depends on a good state of the index, so if there are inaccurancies in the index, make sure to run -"rebuild-index" before! +"repair index" before! WARNING: @@ -66,12 +61,10 @@ type RepairOptions struct { var repairSnapshotOptions RepairOptions func init() { - cmdRoot.AddCommand(cmdRepair) cmdRepair.AddCommand(cmdRepairSnapshots) flags := cmdRepairSnapshots.Flags() initMultiSnapshotFilter(flags, &repairSnapshotOptions.SnapshotFilter, true) - flags.StringVar(&repairSnapshotOptions.AddTag, "add-tag", "repaired", "tag to add to repaired snapshots") flags.StringVar(&repairSnapshotOptions.Append, "append", ".repaired", "string to append to repaired dirs/files; remove files if empty or impossible to repair") flags.BoolVarP(&repairSnapshotOptions.DryRun, "dry-run", "n", true, "don't do anything, only show what would be done") diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 10ebbaf13..42fd26d6b 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -188,7 +188,7 @@ func testRunRebuildIndex(t testing.TB, gopts GlobalOptions) { globalOptions.stdout = os.Stdout }() - rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{}, gopts)) + rtest.OK(t, runRebuildIndex(context.TODO(), RepairIndexOptions{}, gopts)) } func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string { @@ -1504,8 +1504,8 @@ func testRebuildIndex(t *testing.T, backendTestHook backendWrapper) { t.Fatalf("expected no error from checker for test repository, got %v", err) } - if !strings.Contains(out, "restic rebuild-index") { - t.Fatalf("did not find hint for rebuild-index command") + if !strings.Contains(out, "restic repair index") { + t.Fatalf("did not find hint for repair index command") } env.gopts.backendTestHook = backendTestHook @@ -1518,7 +1518,7 @@ func testRebuildIndex(t *testing.T, backendTestHook backendWrapper) { } if err != nil { - t.Fatalf("expected no error from checker after rebuild-index, got: %v", err) + t.Fatalf("expected no error from checker after repair index, got: %v", err) } } @@ -1599,7 +1599,7 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) { env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return &appendOnlyBackend{r}, nil } - err := runRebuildIndex(context.TODO(), RebuildIndexOptions{}, env.gopts) + err := runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts) if err == nil { t.Error("expected rebuildIndex to fail") } @@ -1887,8 +1887,8 @@ func TestListOnce(t *testing.T) { testRunPrune(t, env.gopts, pruneOpts) rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil)) - rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{}, env.gopts)) - rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{ReadAllPacks: true}, env.gopts)) + rtest.OK(t, runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts)) + rtest.OK(t, runRebuildIndex(context.TODO(), RepairIndexOptions{ReadAllPacks: true}, env.gopts)) } func TestHardLink(t *testing.T) { diff --git a/doc/060_forget.rst b/doc/060_forget.rst index 2353ef6a0..72c7ae97f 100644 --- a/doc/060_forget.rst +++ b/doc/060_forget.rst @@ -472,7 +472,7 @@ space. However, a **failed** ``prune`` run can cause the repository to become **temporarily unusable**. Therefore, make sure that you have a stable connection to the repository storage, before running this command. In case the command fails, it may become necessary to manually remove all files from the `index/` folder of the repository and -run `rebuild-index` afterwards. +run `repair index` afterwards. To prevent accidental usages of the ``--unsafe-recover-no-free-space`` option it is necessary to first run ``prune --unsafe-recover-no-free-space SOME-ID`` and then replace diff --git a/doc/077_troubleshooting.rst b/doc/077_troubleshooting.rst index 50c19565e..5b86ffd87 100644 --- a/doc/077_troubleshooting.rst +++ b/doc/077_troubleshooting.rst @@ -58,17 +58,17 @@ But make sure that your needed data is also still contained in your repository ; Note that `check` also prints out warning in some cases. These warnings point out that the repo may be optimized but is still in perfect shape and does not need any troubleshooting. -3. Index trouble -> `rebuild-index` +3. Index trouble -> `repair index` ******************************************** A common problem with broken repostories is that the index does no longer correctly represent the contents of your pack files. This is especially the case if some pack files got lost. -`rebuild-index` recovers this situation and ensures that the index exactly represents the pack files. +`repair index` recovers this situation and ensures that the index exactly represents the pack files. You might even need to manually remove corrupted pack files. In this case make sure, you run -`restic rebuild-index` after. +`restic repair index` after. -Also if you encounter problems with the index files itselves, `rebuild-index` will solve these problems +Also if you encounter problems with the index files itselves, `repair index` will solve these problems immediately. However, rebuilding the index does not solve every problem, e.g. lost pack files. @@ -91,7 +91,7 @@ backup again and check if this did heal your repository. If you realize that a specific file is broken in your repository and you have this file, any run of `backup` which includes that file will be able to heal the situation. -Note that `backup` relies on a correct index state, so make sure your index is fine or run `rebuild-index` +Note that `backup` relies on a correct index state, so make sure your index is fine or run `repair index` before running `backup`. 6. Unreferenced tree -> `recover` @@ -101,7 +101,7 @@ If for some reason you have unreferenced trees in your repository but you actual `recover` it will generate a new snapshot which allows access to all trees that you have in your repository. -Note that `recover` relies on a correct index state, so make sure your index is fine or run `rebuild-index` +Note that `recover` relies on a correct index state, so make sure your index is fine or run `repair index` before running `recover`. 7. Repair defect snapshots using `repair` diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index f812e3a70..093144722 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -35,8 +35,8 @@ Usage help is available: migrate Apply migrations mount Mount the repository prune Remove unneeded data from the repository - rebuild-index Build a new index recover Recover data from the repository not referenced by snapshots + repair Repair the repository restore Extract the data from a snapshot rewrite Rewrite snapshots to exclude unwanted files self-update Update the restic binary diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index a56965d63..3c1cc33d0 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -207,7 +207,7 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error { if arch.Repo.Index().Has(restic.BlobHandle{ID: id, Type: restic.TreeBlob}) { err = errors.Errorf("tree %v could not be loaded; the repository could be damaged: %v", id, err) } else { - err = errors.Errorf("tree %v is not known; the repository could be damaged, run `rebuild-index` to try to repair it", id) + err = errors.Errorf("tree %v is not known; the repository could be damaged, run `repair index` to try to repair it", id) } return err } From 903651c719ebae82b8c25fe29f7bf78a7400ab2f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Tue, 27 Dec 2022 20:24:49 +0100 Subject: [PATCH 103/127] repair snapshots: partially synchronize code with rewrite command Simplify CLI options: * Rename "DeleteSnapshots" to "Forget" * Replace "AddTag" and "Append" with hardcoded values Change output and snapshot modifications to be more in line with the "rewrite" command. --- cmd/restic/cmd_repair_snapshots.go | 62 +++++++++++++++--------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index 8b9005900..8a51d72be 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -3,6 +3,7 @@ package main import ( "context" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" "golang.org/x/sync/errgroup" @@ -50,12 +51,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er // RepairOptions collects all options for the repair command. type RepairOptions struct { - restic.SnapshotFilter + DryRun bool + Forget bool - AddTag string - Append string - DryRun bool - DeleteSnapshots bool + restic.SnapshotFilter } var repairSnapshotOptions RepairOptions @@ -64,28 +63,31 @@ func init() { cmdRepair.AddCommand(cmdRepairSnapshots) flags := cmdRepairSnapshots.Flags() + flags.BoolVarP(&repairSnapshotOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done") + flags.BoolVarP(&repairSnapshotOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones") + initMultiSnapshotFilter(flags, &repairSnapshotOptions.SnapshotFilter, true) - flags.StringVar(&repairSnapshotOptions.AddTag, "add-tag", "repaired", "tag to add to repaired snapshots") - flags.StringVar(&repairSnapshotOptions.Append, "append", ".repaired", "string to append to repaired dirs/files; remove files if empty or impossible to repair") - flags.BoolVarP(&repairSnapshotOptions.DryRun, "dry-run", "n", true, "don't do anything, only show what would be done") - flags.BoolVar(&repairSnapshotOptions.DeleteSnapshots, "delete-snapshots", false, "delete original snapshots") } func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOptions, args []string) error { - switch { - case opts.DryRun: - Printf("\n note: --dry-run is set\n-> repair will only show what it would do.\n\n") - case opts.DeleteSnapshots: - Printf("\n note: --dry-run is not set and --delete-snapshots is set\n-> this may result in data loss!\n\n") - } - repo, err := OpenRepository(ctx, globalOptions) if err != nil { return err } - lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) + if !opts.DryRun { + var lock *restic.Lock + var err error + lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) + defer unlockRepo(lock) + if err != nil { + return err + } + } else { + repo.SetDryRun() + } + + snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) if err != nil { return err } @@ -96,7 +98,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt // get snapshots to check & repair var snapshots []*restic.Snapshot - for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) { + for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) { snapshots = append(snapshots, sn) } @@ -124,7 +126,7 @@ func repairSnapshots(ctx context.Context, opts RepairOptions, repo restic.Reposi Printf("the root tree is damaged -> delete snapshot.\n") deleteSn.Insert(*sn.ID()) case changed: - err = changeSnapshot(ctx, opts, repo, sn, newID) + err = changeSnapshot(ctx, opts.DryRun, repo, sn, newID) if err != nil { return err } @@ -144,7 +146,7 @@ func repairSnapshots(ctx context.Context, opts RepairOptions, repo restic.Reposi return err } - if len(deleteSn) > 0 && opts.DeleteSnapshots { + if len(deleteSn) > 0 && opts.Forget { Verbosef("delete %d snapshots...\n", len(deleteSn)) if !opts.DryRun { DeleteFiles(ctx, globalOptions, repo, deleteSn, restic.SnapshotFile) @@ -158,14 +160,12 @@ func repairSnapshots(ctx context.Context, opts RepairOptions, repo restic.Reposi // - add the rag opts.AddTag // - preserve original ID // if opts.DryRun is set, it doesn't change anything but only -func changeSnapshot(ctx context.Context, opts RepairOptions, repo restic.Repository, sn *restic.Snapshot, newID *restic.ID) error { - sn.AddTags([]string{opts.AddTag}) - // Retain the original snapshot id over all tag changes. - if sn.Original == nil { - sn.Original = sn.ID() - } +func changeSnapshot(ctx context.Context, dryRun bool, repo restic.Repository, sn *restic.Snapshot, newID *restic.ID) error { + sn.AddTags([]string{"repaired"}) + // Always set the original snapshot id as this essentially a new snapshot. + sn.Original = sn.ID() sn.Tree = newID - if !opts.DryRun { + if !dryRun { newID, err := restic.SaveSnapshot(ctx, repo, sn) if err != nil { return err @@ -236,12 +236,12 @@ func repairTree(ctx context.Context, opts RepairOptions, repo restic.Repository, } if !ok { changed = true - if opts.Append == "" || newSize == 0 { + if newSize == 0 { Printf("removed defective file '%v'\n", path+node.Name) continue } Printf("repaired defective file '%v'", path+node.Name) - node.Name = node.Name + opts.Append + node.Name = node.Name + ".repaired" Printf(" to '%v'\n", node.Name) node.Content = newContent node.Size = newSize @@ -256,7 +256,7 @@ func repairTree(ctx context.Context, opts RepairOptions, repo restic.Repository, // If we get an error, we remove this subtree changed = true Printf("removed defective dir '%v'", path+node.Name) - node.Name = node.Name + opts.Append + node.Name = node.Name + ".repaired" Printf("(now empty '%v')\n", node.Name) empty, err := emptyTree(ctx, repo, opts.DryRun) if err != nil { From 375189488cf5e391b9aba1405803398dd3d7daa3 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Tue, 27 Dec 2022 21:05:21 +0100 Subject: [PATCH 104/127] rewrite: prepare for code sharing with rewrite snapshots --- cmd/restic/cmd_rewrite.go | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index 744686390..4019d9264 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -87,22 +87,31 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti return true } + return filterAndReplaceSnapshot(ctx, repo, sn, + func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { + return walker.FilterTree(ctx, repo, "/", *sn.Tree, &walker.TreeFilterVisitor{ + SelectByName: selectByName, + PrintExclude: func(path string) { Verbosef(fmt.Sprintf("excluding %s\n", path)) }, + }) + }, opts.DryRun, opts.Forget, "rewrite") +} + +func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, filter func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error), dryRun bool, forget bool, addTag string) (bool, error) { + wg, wgCtx := errgroup.WithContext(ctx) repo.StartPackUploader(wgCtx, wg) var filteredTree restic.ID wg.Go(func() error { - filteredTree, err = walker.FilterTree(wgCtx, repo, "/", *sn.Tree, &walker.TreeFilterVisitor{ - SelectByName: selectByName, - PrintExclude: func(path string) { Verbosef(fmt.Sprintf("excluding %s\n", path)) }, - }) + var err error + filteredTree, err = filter(ctx, sn) if err != nil { return err } return repo.Flush(wgCtx) }) - err = wg.Wait() + err := wg.Wait() if err != nil { return false, err } @@ -113,10 +122,10 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti } debug.Log("Snapshot %v modified", sn) - if opts.DryRun { + if dryRun { Verbosef("would save new snapshot\n") - if opts.Forget { + if forget { Verbosef("would remove old snapshot\n") } @@ -125,10 +134,10 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti // Always set the original snapshot id as this essentially a new snapshot. sn.Original = sn.ID() - *sn.Tree = filteredTree + sn.Tree = &filteredTree - if !opts.Forget { - sn.AddTags([]string{"rewrite"}) + if !forget { + sn.AddTags([]string{addTag}) } // Save the new snapshot. @@ -138,7 +147,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti } Verbosef("saved new snapshot %v\n", id.Str()) - if opts.Forget { + if forget { h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} if err = repo.Backend().Remove(ctx, h); err != nil { return false, err From 8c4caf09a82a5b8adc540d92abc68194051af679 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Tue, 27 Dec 2022 21:34:07 +0100 Subject: [PATCH 105/127] repair snapshots: Do not rename repaired files The files in a tree must be sorted in lexical order. However, this cannot be guaranteed when appending a filename suffix. For two files file, file.rep where "file" is broken, this would result in file.repaired, file.rep which is no longer sorted. In addition, adding a filename suffix is also prone to filename collisions which would require a rather complex search for a collision-free name in order to work reliably. --- cmd/restic/cmd_repair_snapshots.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index 8a51d72be..0413c43bf 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -240,9 +240,7 @@ func repairTree(ctx context.Context, opts RepairOptions, repo restic.Repository, Printf("removed defective file '%v'\n", path+node.Name) continue } - Printf("repaired defective file '%v'", path+node.Name) - node.Name = node.Name + ".repaired" - Printf(" to '%v'\n", node.Name) + Printf("repaired defective file '%v'\n", path+node.Name) node.Content = newContent node.Size = newSize } @@ -255,9 +253,7 @@ func repairTree(ctx context.Context, opts RepairOptions, repo restic.Repository, case lErr: // If we get an error, we remove this subtree changed = true - Printf("removed defective dir '%v'", path+node.Name) - node.Name = node.Name + ".repaired" - Printf("(now empty '%v')\n", node.Name) + Printf("replaced defective dir '%v'", path+node.Name) empty, err := emptyTree(ctx, repo, opts.DryRun) if err != nil { return newID, true, false, err From 1a9705fc957fe1fedd709da5bbf016a5b2c4d98f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 28 Dec 2022 10:38:40 +0100 Subject: [PATCH 106/127] walker: Simplify change detection in FilterTree Now the rewritten tree is always serialized which makes sure that we don't accidentally miss any relevant changes. --- internal/walker/rewriter.go | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/internal/walker/rewriter.go b/internal/walker/rewriter.go index cd05f69f5..96afbb07e 100644 --- a/internal/walker/rewriter.go +++ b/internal/walker/rewriter.go @@ -42,7 +42,6 @@ func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID debug.Log("filterTree: %s, nodeId: %s\n", nodepath, nodeID.Str()) - changed := false tb := restic.NewTreeJSONBuilder() for _, node := range curTree.Nodes { path := path.Join(nodepath, node.Name) @@ -50,7 +49,6 @@ func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID if visitor.PrintExclude != nil { visitor.PrintExclude(path) } - changed = true continue } @@ -65,9 +63,6 @@ func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID if err != nil { return restic.ID{}, err } - if !node.Subtree.Equal(newID) { - changed = true - } node.Subtree = &newID err = tb.AddNode(node) if err != nil { @@ -75,17 +70,15 @@ func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID } } - if changed { - tree, err := tb.Finalize() - if err != nil { - return restic.ID{}, err - } - - // Save new tree - newTreeID, _, _, err := repo.SaveBlob(ctx, restic.TreeBlob, tree, restic.ID{}, false) - debug.Log("filterTree: save new tree for %s as %v\n", nodepath, newTreeID) - return newTreeID, err + tree, err := tb.Finalize() + if err != nil { + return restic.ID{}, err } - return nodeID, nil + // Save new tree + newTreeID, _, _, err := repo.SaveBlob(ctx, restic.TreeBlob, tree, restic.ID{}, false) + if !newTreeID.Equal(nodeID) { + debug.Log("filterTree: save new tree for %s as %v\n", nodepath, newTreeID) + } + return newTreeID, err } From bc2399fbd9c9951602ae4ed6be9c346625d8face Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 28 Dec 2022 10:42:21 +0100 Subject: [PATCH 107/127] walker: recurse into directory based on node type A broken directory might also not have a subtree. --- internal/walker/rewriter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/walker/rewriter.go b/internal/walker/rewriter.go index 96afbb07e..bef3bd688 100644 --- a/internal/walker/rewriter.go +++ b/internal/walker/rewriter.go @@ -52,7 +52,7 @@ func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID continue } - if node.Subtree == nil { + if node.Type != "dir" { err = tb.AddNode(node) if err != nil { return restic.ID{}, err From 38dac78180db7ae4d49e04cd16d4fb94ddae8c0a Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 28 Dec 2022 11:04:28 +0100 Subject: [PATCH 108/127] walker: restructure FilterTree into TreeRewriter The more generic RewriteNode callback replaces the SelectByName and PrintExclude functions. The main part of this change is a preparation to allow using the TreeRewriter for the `repair snapshots` command. --- cmd/restic/cmd_rewrite.go | 15 +++-- internal/walker/rewriter.go | 37 +++++++++---- internal/walker/rewriter_test.go | 94 +++++++++++++++++++++----------- internal/walker/walker_test.go | 5 +- 4 files changed, 103 insertions(+), 48 deletions(-) diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index 4019d9264..a60fdc8fc 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -87,12 +87,19 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti return true } + rewriter := walker.NewTreeRewriter(walker.RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { + if selectByName(path) { + return node + } + Verbosef(fmt.Sprintf("excluding %s\n", path)) + return nil + }, + }) + return filterAndReplaceSnapshot(ctx, repo, sn, func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { - return walker.FilterTree(ctx, repo, "/", *sn.Tree, &walker.TreeFilterVisitor{ - SelectByName: selectByName, - PrintExclude: func(path string) { Verbosef(fmt.Sprintf("excluding %s\n", path)) }, - }) + return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) }, opts.DryRun, opts.Forget, "rewrite") } diff --git a/internal/walker/rewriter.go b/internal/walker/rewriter.go index bef3bd688..48f16a53a 100644 --- a/internal/walker/rewriter.go +++ b/internal/walker/rewriter.go @@ -9,13 +9,28 @@ import ( "github.com/restic/restic/internal/restic" ) -// SelectByNameFunc returns true for all items that should be included (files and -// dirs). If false is returned, files are ignored and dirs are not even walked. -type SelectByNameFunc func(item string) bool +type NodeRewriteFunc func(node *restic.Node, path string) *restic.Node -type TreeFilterVisitor struct { - SelectByName SelectByNameFunc - PrintExclude func(string) +type RewriteOpts struct { + // return nil to remove the node + RewriteNode NodeRewriteFunc +} + +type TreeRewriter struct { + opts RewriteOpts +} + +func NewTreeRewriter(opts RewriteOpts) *TreeRewriter { + rw := &TreeRewriter{ + opts: opts, + } + // setup default implementations + if rw.opts.RewriteNode == nil { + rw.opts.RewriteNode = func(node *restic.Node, path string) *restic.Node { + return node + } + } + return rw } type BlobLoadSaver interface { @@ -23,7 +38,7 @@ type BlobLoadSaver interface { restic.BlobLoader } -func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID restic.ID, visitor *TreeFilterVisitor) (newNodeID restic.ID, err error) { +func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID restic.ID) (newNodeID restic.ID, err error) { curTree, err := restic.LoadTree(ctx, repo, nodeID) if err != nil { return restic.ID{}, err @@ -45,10 +60,8 @@ func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID tb := restic.NewTreeJSONBuilder() for _, node := range curTree.Nodes { path := path.Join(nodepath, node.Name) - if !visitor.SelectByName(path) { - if visitor.PrintExclude != nil { - visitor.PrintExclude(path) - } + node = t.opts.RewriteNode(node, path) + if node == nil { continue } @@ -59,7 +72,7 @@ func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID } continue } - newID, err := FilterTree(ctx, repo, path, *node.Subtree, visitor) + newID, err := t.RewriteTree(ctx, repo, path, *node.Subtree) if err != nil { return restic.ID{}, err } diff --git a/internal/walker/rewriter_test.go b/internal/walker/rewriter_test.go index 3dcf0ac9e..8f99fe9bd 100644 --- a/internal/walker/rewriter_test.go +++ b/internal/walker/rewriter_test.go @@ -5,7 +5,6 @@ import ( "fmt" "testing" - "github.com/google/go-cmp/cmp" "github.com/pkg/errors" "github.com/restic/restic/internal/restic" ) @@ -38,26 +37,26 @@ func (t WritableTreeMap) Dump() { } } -type checkRewriteFunc func(t testing.TB) (visitor TreeFilterVisitor, final func(testing.TB)) +type checkRewriteFunc func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) // checkRewriteItemOrder ensures that the order of the 'path' arguments is the one passed in as 'want'. func checkRewriteItemOrder(want []string) checkRewriteFunc { pos := 0 - return func(t testing.TB) (visitor TreeFilterVisitor, final func(testing.TB)) { - vis := TreeFilterVisitor{ - SelectByName: func(path string) bool { + return func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) { + rewriter = NewTreeRewriter(RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { if pos >= len(want) { t.Errorf("additional unexpected path found: %v", path) - return false + return nil } if path != want[pos] { t.Errorf("wrong path found, want %q, got %q", want[pos], path) } pos++ - return true + return node }, - } + }) final = func(t testing.TB) { if pos != len(want) { @@ -65,21 +64,20 @@ func checkRewriteItemOrder(want []string) checkRewriteFunc { } } - return vis, final + return rewriter, final } } -// checkRewriteSkips excludes nodes if path is in skipFor, it checks that all excluded entries are printed. +// checkRewriteSkips excludes nodes if path is in skipFor, it checks that rewriting proceedes in the correct order. func checkRewriteSkips(skipFor map[string]struct{}, want []string) checkRewriteFunc { var pos int - printed := make(map[string]struct{}) - return func(t testing.TB) (visitor TreeFilterVisitor, final func(testing.TB)) { - vis := TreeFilterVisitor{ - SelectByName: func(path string) bool { + return func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) { + rewriter = NewTreeRewriter(RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { if pos >= len(want) { t.Errorf("additional unexpected path found: %v", path) - return false + return nil } if path != want[pos] { @@ -87,27 +85,39 @@ func checkRewriteSkips(skipFor map[string]struct{}, want []string) checkRewriteF } pos++ - _, ok := skipFor[path] - return !ok - }, - PrintExclude: func(s string) { - if _, ok := printed[s]; ok { - t.Errorf("path was already printed %v", s) + _, skip := skipFor[path] + if skip { + return nil } - printed[s] = struct{}{} + return node }, - } + }) final = func(t testing.TB) { - if !cmp.Equal(skipFor, printed) { - t.Errorf("unexpected paths skipped: %s", cmp.Diff(skipFor, printed)) - } if pos != len(want) { t.Errorf("not enough items returned, want %d, got %d", len(want), pos) } } - return vis, final + return rewriter, final + } +} + +// checkIncreaseNodeSize modifies each node by changing its size. +func checkIncreaseNodeSize(increase uint64) checkRewriteFunc { + return func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) { + rewriter = NewTreeRewriter(RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { + if node.Type == "file" { + node.Size += increase + } + return node + }, + }) + + final = func(t testing.TB) {} + + return rewriter, final } } @@ -172,6 +182,21 @@ func TestRewriter(t *testing.T) { }, ), }, + { // modify node + tree: TestTree{ + "foo": TestFile{Size: 21}, + "subdir": TestTree{ + "subfile": TestFile{Size: 21}, + }, + }, + newTree: TestTree{ + "foo": TestFile{Size: 42}, + "subdir": TestTree{ + "subfile": TestFile{Size: 42}, + }, + }, + check: checkIncreaseNodeSize(21), + }, } for _, test := range tests { @@ -186,8 +211,8 @@ func TestRewriter(t *testing.T) { ctx, cancel := context.WithCancel(context.TODO()) defer cancel() - vis, last := test.check(t) - newRoot, err := FilterTree(ctx, modrepo, "/", root, &vis) + rewriter, last := test.check(t) + newRoot, err := rewriter.RewriteTree(ctx, modrepo, "/", root) if err != nil { t.Error(err) } @@ -213,8 +238,15 @@ func TestRewriterFailOnUnknownFields(t *testing.T) { ctx, cancel := context.WithCancel(context.TODO()) defer cancel() - // use nil visitor to crash if the tree loading works unexpectedly - _, err := FilterTree(ctx, tm, "/", id, nil) + + rewriter := NewTreeRewriter(RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { + // tree loading must not succeed + t.Fail() + return node + }, + }) + _, err := rewriter.RewriteTree(ctx, tm, "/", id) if err == nil { t.Error("missing error on unknown field") diff --git a/internal/walker/walker_test.go b/internal/walker/walker_test.go index 6c4fd3436..8de1a9dc4 100644 --- a/internal/walker/walker_test.go +++ b/internal/walker/walker_test.go @@ -14,7 +14,9 @@ import ( type TestTree map[string]interface{} // TestNode is used to test the walker. -type TestFile struct{} +type TestFile struct { + Size uint64 +} func BuildTreeMap(tree TestTree) (m TreeMap, root restic.ID) { m = TreeMap{} @@ -37,6 +39,7 @@ func buildTreeMap(tree TestTree, m TreeMap) restic.ID { err := tb.AddNode(&restic.Node{ Name: name, Type: "file", + Size: elem.Size, }) if err != nil { panic(err) From 1bd1f3008ddf74f6f702c1c9c2b7c86d938e83f6 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 28 Dec 2022 11:34:55 +0100 Subject: [PATCH 109/127] walker: extend TreeRewriter to support snapshot repairing This adds support for caching already rewritten trees, handling of load errors and disabling the check that the serialization doesn't lead to data loss. --- cmd/restic/cmd_rewrite.go | 1 + internal/walker/rewriter.go | 58 +++++++++++++--- internal/walker/rewriter_test.go | 114 ++++++++++++++++++++++++++++++- 3 files changed, 161 insertions(+), 12 deletions(-) diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index a60fdc8fc..e5c65850d 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -95,6 +95,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti Verbosef(fmt.Sprintf("excluding %s\n", path)) return nil }, + DisableNodeCache: true, }) return filterAndReplaceSnapshot(ctx, repo, sn, diff --git a/internal/walker/rewriter.go b/internal/walker/rewriter.go index 48f16a53a..649857032 100644 --- a/internal/walker/rewriter.go +++ b/internal/walker/rewriter.go @@ -10,26 +10,45 @@ import ( ) type NodeRewriteFunc func(node *restic.Node, path string) *restic.Node +type FailedTreeRewriteFunc func(nodeID restic.ID, path string, err error) (restic.ID, error) type RewriteOpts struct { // return nil to remove the node RewriteNode NodeRewriteFunc + // decide what to do with a tree that could not be loaded. Return nil to remove the node. By default the load error is returned which causes the operation to fail. + RewriteFailedTree FailedTreeRewriteFunc + + AllowUnstableSerialization bool + DisableNodeCache bool } +type idMap map[restic.ID]restic.ID + type TreeRewriter struct { opts RewriteOpts + + replaces idMap } func NewTreeRewriter(opts RewriteOpts) *TreeRewriter { rw := &TreeRewriter{ opts: opts, } + if !opts.DisableNodeCache { + rw.replaces = make(idMap) + } // setup default implementations if rw.opts.RewriteNode == nil { rw.opts.RewriteNode = func(node *restic.Node, path string) *restic.Node { return node } } + if rw.opts.RewriteFailedTree == nil { + // fail with error by default + rw.opts.RewriteFailedTree = func(nodeID restic.ID, path string, err error) (restic.ID, error) { + return restic.ID{}, err + } + } return rw } @@ -39,20 +58,29 @@ type BlobLoadSaver interface { } func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID restic.ID) (newNodeID restic.ID, err error) { - curTree, err := restic.LoadTree(ctx, repo, nodeID) - if err != nil { - return restic.ID{}, err + // check if tree was already changed + newID, ok := t.replaces[nodeID] + if ok { + return newID, nil } - // check that we can properly encode this tree without losing information - // The alternative of using json/Decoder.DisallowUnknownFields() doesn't work as we use - // a custom UnmarshalJSON to decode trees, see also https://github.com/golang/go/issues/41144 - testID, err := restic.SaveTree(ctx, repo, curTree) + // a nil nodeID will lead to a load error + curTree, err := restic.LoadTree(ctx, repo, nodeID) if err != nil { - return restic.ID{}, err + return t.opts.RewriteFailedTree(nodeID, nodepath, err) } - if nodeID != testID { - return restic.ID{}, fmt.Errorf("cannot encode tree at %q without losing information", nodepath) + + if !t.opts.AllowUnstableSerialization { + // check that we can properly encode this tree without losing information + // The alternative of using json/Decoder.DisallowUnknownFields() doesn't work as we use + // a custom UnmarshalJSON to decode trees, see also https://github.com/golang/go/issues/41144 + testID, err := restic.SaveTree(ctx, repo, curTree) + if err != nil { + return restic.ID{}, err + } + if nodeID != testID { + return restic.ID{}, fmt.Errorf("cannot encode tree at %q without losing information", nodepath) + } } debug.Log("filterTree: %s, nodeId: %s\n", nodepath, nodeID.Str()) @@ -72,7 +100,12 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, node } continue } - newID, err := t.RewriteTree(ctx, repo, path, *node.Subtree) + // treat nil as null id + var subtree restic.ID + if node.Subtree != nil { + subtree = *node.Subtree + } + newID, err := t.RewriteTree(ctx, repo, path, subtree) if err != nil { return restic.ID{}, err } @@ -90,6 +123,9 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, node // Save new tree newTreeID, _, _, err := repo.SaveBlob(ctx, restic.TreeBlob, tree, restic.ID{}, false) + if t.replaces != nil { + t.replaces[nodeID] = newTreeID + } if !newTreeID.Equal(nodeID) { debug.Log("filterTree: save new tree for %s as %v\n", nodepath, newTreeID) } diff --git a/internal/walker/rewriter_test.go b/internal/walker/rewriter_test.go index 8f99fe9bd..07ce5f72f 100644 --- a/internal/walker/rewriter_test.go +++ b/internal/walker/rewriter_test.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" ) // WritableTreeMap also support saving @@ -69,7 +70,7 @@ func checkRewriteItemOrder(want []string) checkRewriteFunc { } // checkRewriteSkips excludes nodes if path is in skipFor, it checks that rewriting proceedes in the correct order. -func checkRewriteSkips(skipFor map[string]struct{}, want []string) checkRewriteFunc { +func checkRewriteSkips(skipFor map[string]struct{}, want []string, disableCache bool) checkRewriteFunc { var pos int return func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) { @@ -91,6 +92,7 @@ func checkRewriteSkips(skipFor map[string]struct{}, want []string) checkRewriteF } return node }, + DisableNodeCache: disableCache, }) final = func(t testing.TB) { @@ -160,6 +162,7 @@ func TestRewriter(t *testing.T) { "/subdir", "/subdir/subfile", }, + false, ), }, { // exclude dir @@ -180,6 +183,7 @@ func TestRewriter(t *testing.T) { "/foo", "/subdir", }, + false, ), }, { // modify node @@ -197,6 +201,75 @@ func TestRewriter(t *testing.T) { }, check: checkIncreaseNodeSize(21), }, + { // test cache + tree: TestTree{ + // both subdirs are identical + "subdir1": TestTree{ + "subfile": TestFile{}, + "subfile2": TestFile{}, + }, + "subdir2": TestTree{ + "subfile": TestFile{}, + "subfile2": TestFile{}, + }, + }, + newTree: TestTree{ + "subdir1": TestTree{ + "subfile2": TestFile{}, + }, + "subdir2": TestTree{ + "subfile2": TestFile{}, + }, + }, + check: checkRewriteSkips( + map[string]struct{}{ + "/subdir1/subfile": {}, + }, + []string{ + "/subdir1", + "/subdir1/subfile", + "/subdir1/subfile2", + "/subdir2", + }, + false, + ), + }, + { // test disabled cache + tree: TestTree{ + // both subdirs are identical + "subdir1": TestTree{ + "subfile": TestFile{}, + "subfile2": TestFile{}, + }, + "subdir2": TestTree{ + "subfile": TestFile{}, + "subfile2": TestFile{}, + }, + }, + newTree: TestTree{ + "subdir1": TestTree{ + "subfile2": TestFile{}, + }, + "subdir2": TestTree{ + "subfile": TestFile{}, + "subfile2": TestFile{}, + }, + }, + check: checkRewriteSkips( + map[string]struct{}{ + "/subdir1/subfile": {}, + }, + []string{ + "/subdir1", + "/subdir1/subfile", + "/subdir1/subfile2", + "/subdir2", + "/subdir2/subfile", + "/subdir2/subfile2", + }, + true, + ), + }, } for _, test := range tests { @@ -251,4 +324,43 @@ func TestRewriterFailOnUnknownFields(t *testing.T) { if err == nil { t.Error("missing error on unknown field") } + + // check that the serialization check can be disabled + rewriter = NewTreeRewriter(RewriteOpts{ + AllowUnstableSerialization: true, + }) + root, err := rewriter.RewriteTree(ctx, tm, "/", id) + test.OK(t, err) + _, expRoot := BuildTreeMap(TestTree{ + "subfile": TestFile{}, + }) + test.Assert(t, root == expRoot, "mismatched trees") +} + +func TestRewriterTreeLoadError(t *testing.T) { + tm := WritableTreeMap{TreeMap{}} + id := restic.NewRandomID() + + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + + // also check that load error by default cause the operation to fail + rewriter := NewTreeRewriter(RewriteOpts{}) + _, err := rewriter.RewriteTree(ctx, tm, "/", id) + if err == nil { + t.Fatal("missing error on unloadable tree") + } + + replacementID := restic.NewRandomID() + rewriter = NewTreeRewriter(RewriteOpts{ + RewriteFailedTree: func(nodeID restic.ID, path string, err error) (restic.ID, error) { + if nodeID != id || path != "/" { + t.Fail() + } + return replacementID, nil + }, + }) + newRoot, err := rewriter.RewriteTree(ctx, tm, "/", id) + test.OK(t, err) + test.Equals(t, replacementID, newRoot) } From e17ee40a31e8eb9004c300b346fe4d054cb45318 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 28 Dec 2022 11:50:02 +0100 Subject: [PATCH 110/127] repair snapshots: Port to use walker.TreeRewriter --- cmd/restic/cmd_repair_snapshots.go | 180 +++++++++-------------------- 1 file changed, 56 insertions(+), 124 deletions(-) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index 0413c43bf..ac75637fd 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -6,6 +6,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/walker" "golang.org/x/sync/errgroup" "github.com/spf13/cobra" @@ -106,8 +107,56 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt } func repairSnapshots(ctx context.Context, opts RepairOptions, repo restic.Repository, snapshots []*restic.Snapshot) error { - replaces := make(idMap) - seen := restic.NewIDSet() + // Three error cases are checked: + // - tree is a nil tree (-> will be replaced by an empty tree) + // - trees which cannot be loaded (-> the tree contents will be removed) + // - files whose contents are not fully available (-> file will be modified) + rewriter := walker.NewTreeRewriter(walker.RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { + if node.Type != "file" { + return node + } + + ok := true + var newContent restic.IDs + var newSize uint64 + // check all contents and remove if not available + for _, id := range node.Content { + if size, found := repo.LookupBlobSize(id, restic.DataBlob); !found { + ok = false + } else { + newContent = append(newContent, id) + newSize += uint64(size) + } + } + if !ok { + if newSize == 0 { + Printf("removed defective file '%v'\n", path+node.Name) + node = nil + } else { + Printf("repaired defective file '%v'\n", path+node.Name) + node.Content = newContent + node.Size = newSize + } + } + return node + }, + RewriteFailedTree: func(nodeID restic.ID, path string, _ error) (restic.ID, error) { + if path == "/" { + // remove snapshots with invalid root node + return restic.ID{}, nil + } + // If a subtree fails to load, remove it + Printf("removed defective dir '%v'", path) + emptyID, err := restic.SaveTree(ctx, repo, &restic.Tree{}) + if err != nil { + return restic.ID{}, err + } + return emptyID, nil + }, + AllowUnstableSerialization: true, + }) + deleteSn := restic.NewIDSet() Verbosef("check and repair %d snapshots\n", len(snapshots)) @@ -118,15 +167,16 @@ func repairSnapshots(ctx context.Context, opts RepairOptions, repo restic.Reposi for _, sn := range snapshots { debug.Log("process snapshot %v", sn.ID()) Printf("%v:\n", sn) - newID, changed, lErr, err := repairTree(ctx, opts, repo, "/", sn.Tree, replaces, seen) + newID, err := rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) + switch { case err != nil: return err - case lErr: + case newID.IsNull(): Printf("the root tree is damaged -> delete snapshot.\n") deleteSn.Insert(*sn.ID()) - case changed: - err = changeSnapshot(ctx, opts.DryRun, repo, sn, newID) + case !newID.Equal(*sn.Tree): + err = changeSnapshot(ctx, opts.DryRun, repo, sn, &newID) if err != nil { return err } @@ -176,121 +226,3 @@ func changeSnapshot(ctx context.Context, dryRun bool, repo restic.Repository, sn } return nil } - -type idMap map[restic.ID]restic.ID - -// repairTree checks and repairs a tree and all its subtrees -// Three error cases are checked: -// - tree is a nil tree (-> will be replaced by an empty tree) -// - trees which cannot be loaded (-> the tree contents will be removed) -// - files whose contents are not fully available (-> file will be modified) -// In case of an error, the changes made depends on: -// - opts.Append: string to append to "repared" names; if empty files will not repaired but deleted -// - opts.DryRun: if set to true, only print out what to but don't change anything -// Returns: -// - the new ID -// - whether the ID changed -// - whether there was a load error when loading this tre -// - error for other errors (these are errors when saving a tree) -func repairTree(ctx context.Context, opts RepairOptions, repo restic.Repository, path string, treeID *restic.ID, replaces idMap, seen restic.IDSet) (*restic.ID, bool, bool, error) { - // handle and repair nil trees - if treeID == nil { - empty, err := emptyTree(ctx, repo, opts.DryRun) - Printf("repaired nil tree '%v'\n", path) - return &empty, true, false, err - } - - // check if tree was already changed - newID, ok := replaces[*treeID] - if ok { - return &newID, true, false, nil - } - - // check if tree was seen but not changed - if seen.Has(*treeID) { - return treeID, false, false, nil - } - - tree, err := restic.LoadTree(ctx, repo, *treeID) - if err != nil { - // mark as load error - return &newID, false, true, nil - } - - var newNodes []*restic.Node - changed := false - for _, node := range tree.Nodes { - switch node.Type { - case "file": - ok := true - var newContent restic.IDs - var newSize uint64 - // check all contents and remove if not available - for _, id := range node.Content { - if size, found := repo.LookupBlobSize(id, restic.DataBlob); !found { - ok = false - } else { - newContent = append(newContent, id) - newSize += uint64(size) - } - } - if !ok { - changed = true - if newSize == 0 { - Printf("removed defective file '%v'\n", path+node.Name) - continue - } - Printf("repaired defective file '%v'\n", path+node.Name) - node.Content = newContent - node.Size = newSize - } - case "dir": - // rewrite if necessary - newID, c, lErr, err := repairTree(ctx, opts, repo, path+node.Name+"/", node.Subtree, replaces, seen) - switch { - case err != nil: - return newID, true, false, err - case lErr: - // If we get an error, we remove this subtree - changed = true - Printf("replaced defective dir '%v'", path+node.Name) - empty, err := emptyTree(ctx, repo, opts.DryRun) - if err != nil { - return newID, true, false, err - } - node.Subtree = &empty - case c: - node.Subtree = newID - changed = true - } - } - newNodes = append(newNodes, node) - } - - if !changed { - seen.Insert(*treeID) - return treeID, false, false, nil - } - - tree.Nodes = newNodes - - if !opts.DryRun { - newID, err = restic.SaveTree(ctx, repo, tree) - if err != nil { - return &newID, true, false, err - } - Printf("modified tree %v, new id: %v\n", treeID.Str(), newID.Str()) - } else { - Printf("would have modified tree %v\n", treeID.Str()) - } - - replaces[*treeID] = newID - return &newID, true, false, nil -} - -func emptyTree(ctx context.Context, repo restic.Repository, dryRun bool) (restic.ID, error) { - if !dryRun { - return restic.SaveTree(ctx, repo, &restic.Tree{}) - } - return restic.ID{}, nil -} From 4ce87a7f6487034cfdf113965801a2b02c447d43 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Tue, 27 Dec 2022 21:31:04 +0100 Subject: [PATCH 111/127] repair snapshots: port to filterAndReplaceSnapshot The previous approach of rewriting all snapshots first, then flushing the repository data and finally removing old snapshots has the downside that an interrupted command execution leaves behind broken snapshots as not all new data is already flushed. --- cmd/restic/cmd_repair_snapshots.go | 104 ++++++++--------------------- cmd/restic/cmd_rewrite.go | 14 ++++ 2 files changed, 42 insertions(+), 76 deletions(-) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index ac75637fd..036338161 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -4,10 +4,9 @@ import ( "context" "github.com/restic/restic/internal/backend" - "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/walker" - "golang.org/x/sync/errgroup" "github.com/spf13/cobra" ) @@ -97,16 +96,6 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt return err } - // get snapshots to check & repair - var snapshots []*restic.Snapshot - for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) { - snapshots = append(snapshots, sn) - } - - return repairSnapshots(ctx, opts, repo, snapshots) -} - -func repairSnapshots(ctx context.Context, opts RepairOptions, repo restic.Repository, snapshots []*restic.Snapshot) error { // Three error cases are checked: // - tree is a nil tree (-> will be replaced by an empty tree) // - trees which cannot be loaded (-> the tree contents will be removed) @@ -157,72 +146,35 @@ func repairSnapshots(ctx context.Context, opts RepairOptions, repo restic.Reposi AllowUnstableSerialization: true, }) - deleteSn := restic.NewIDSet() - - Verbosef("check and repair %d snapshots\n", len(snapshots)) - bar := newProgressMax(!globalOptions.Quiet, uint64(len(snapshots)), "snapshots") - wg, ctx := errgroup.WithContext(ctx) - repo.StartPackUploader(ctx, wg) - wg.Go(func() error { - for _, sn := range snapshots { - debug.Log("process snapshot %v", sn.ID()) - Printf("%v:\n", sn) - newID, err := rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) - - switch { - case err != nil: - return err - case newID.IsNull(): - Printf("the root tree is damaged -> delete snapshot.\n") - deleteSn.Insert(*sn.ID()) - case !newID.Equal(*sn.Tree): - err = changeSnapshot(ctx, opts.DryRun, repo, sn, &newID) - if err != nil { - return err - } - deleteSn.Insert(*sn.ID()) - default: - Printf("is ok.\n") - } - debug.Log("processed snapshot %v", sn.ID()) - bar.Add(1) - } - bar.Done() - return repo.Flush(ctx) - }) - - err := wg.Wait() - if err != nil { - return err - } - - if len(deleteSn) > 0 && opts.Forget { - Verbosef("delete %d snapshots...\n", len(deleteSn)) - if !opts.DryRun { - DeleteFiles(ctx, globalOptions, repo, deleteSn, restic.SnapshotFile) - } - } - return nil -} - -// changeSnapshot creates a modified snapshot: -// - set the tree to newID -// - add the rag opts.AddTag -// - preserve original ID -// if opts.DryRun is set, it doesn't change anything but only -func changeSnapshot(ctx context.Context, dryRun bool, repo restic.Repository, sn *restic.Snapshot, newID *restic.ID) error { - sn.AddTags([]string{"repaired"}) - // Always set the original snapshot id as this essentially a new snapshot. - sn.Original = sn.ID() - sn.Tree = newID - if !dryRun { - newID, err := restic.SaveSnapshot(ctx, repo, sn) + changedCount := 0 + for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) { + Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) + changed, err := filterAndReplaceSnapshot(ctx, repo, sn, + func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { + return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) + }, opts.DryRun, opts.Forget, "repaired") if err != nil { - return err + return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err) + } + if changed { + changedCount++ } - Printf("snapshot repaired -> %v created.\n", newID.Str()) - } else { - Printf("would have repaired snapshot %v.\n", sn.ID().Str()) } + + Verbosef("\n") + if changedCount == 0 { + if !opts.DryRun { + Verbosef("no snapshots were modified\n") + } else { + Verbosef("no snapshots would be modified\n") + } + } else { + if !opts.DryRun { + Verbosef("modified %v snapshots\n", changedCount) + } else { + Verbosef("would modify %v snapshots\n", changedCount) + } + } + return nil } diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index e5c65850d..c08797c48 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -124,6 +124,20 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r return false, err } + if filteredTree.IsNull() { + if dryRun { + Verbosef("would delete empty snapshot\n") + } else { + h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} + if err = repo.Backend().Remove(ctx, h); err != nil { + return false, err + } + debug.Log("removed empty snapshot %v", sn.ID()) + Verbosef("removed empty snapshot %v\n", sn.ID().Str()) + } + return true, nil + } + if filteredTree == *sn.Tree { debug.Log("Snapshot %v not modified", sn) return false, nil From f6cc10578ddd9bc15b4c0130477210362c6526a9 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 28 Dec 2022 12:14:33 +0100 Subject: [PATCH 112/127] repair snapshots: Always sanitize file nodes If the node for a file is intact, this is a no-op. --- cmd/restic/cmd_repair_snapshots.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index 036338161..58da8132a 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -107,7 +107,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt } ok := true - var newContent restic.IDs + var newContent restic.IDs = restic.IDs{} var newSize uint64 // check all contents and remove if not available for _, id := range node.Content { @@ -119,15 +119,13 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt } } if !ok { - if newSize == 0 { - Printf("removed defective file '%v'\n", path+node.Name) - node = nil - } else { - Printf("repaired defective file '%v'\n", path+node.Name) - node.Content = newContent - node.Size = newSize - } + Verbosef(" file %q: removed missing content\n", path) + } else if newSize != node.Size { + Verbosef(" file %q: fixed incorrect size\n", path) } + // no-ops if already correct + node.Content = newContent + node.Size = newSize return node }, RewriteFailedTree: func(nodeID restic.ID, path string, _ error) (restic.ID, error) { From 7c8dd61e8c815c44d1ac13fad65214539e47b4f0 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 28 Dec 2022 12:15:39 +0100 Subject: [PATCH 113/127] repair snapshots: cleanup warnings --- cmd/restic/cmd_repair_snapshots.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index 58da8132a..e50d34b9d 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -130,11 +130,12 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt }, RewriteFailedTree: func(nodeID restic.ID, path string, _ error) (restic.ID, error) { if path == "/" { + Verbosef(" dir %q: not readable\n", path) // remove snapshots with invalid root node return restic.ID{}, nil } // If a subtree fails to load, remove it - Printf("removed defective dir '%v'", path) + Verbosef(" dir %q: replaced with empty directory\n", path) emptyID, err := restic.SaveTree(ctx, repo, &restic.Tree{}) if err != nil { return restic.ID{}, err From 9c64a95df80d7f1dfb8f048630e2f432246ab544 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 28 Dec 2022 15:12:39 +0100 Subject: [PATCH 114/127] doc: rewrite troubleshooting section --- doc/077_troubleshooting.rst | 225 ++++++++++++++++++++++++------------ 1 file changed, 154 insertions(+), 71 deletions(-) diff --git a/doc/077_troubleshooting.rst b/doc/077_troubleshooting.rst index 5b86ffd87..fe317acfc 100644 --- a/doc/077_troubleshooting.rst +++ b/doc/077_troubleshooting.rst @@ -14,98 +14,181 @@ Troubleshooting ######################### -Being a backup software, the repository format ensures that the data saved in the repository -is verifiable and error-restistant. Restic even implements some self-healing functionalities. +The repository format used by restic is designed to be error resistant. In +particular, commands like, for example, ``backup`` or ``prune`` can be interrupted +at *any* point in time without damaging the repository. You might have to run +``unlock`` manually though, but that's it. -However, situations might occur where your repository gets in an incorrect state and measurements -need to be done to get you out of this situation. These situations might be due to hardware failure, -accidentially removing files directly from the repository or bugs in the restic implementation. +However, a repository might be damaged if some of its files are damaged or lost. +This can occur due to hardware failures, accidentally removing files from the +repository or bugs in the implementation of restic. -This document is meant to give you some hints about how to recover from such situations. +The following steps will help you recover a repository. This guide does not cover +all possible types of repository damages. Thus, if the steps do not work for you +or you are unsure how to proceed, then ask for help. Please always include the +check output discussed in the next section and what steps you've taken to repair +the repository so far. -1. Stay calm and don't over-react -******************************************** +* `Forum `_ +* Our IRC channel ``#restic`` on ``irc.libera.chat`` -The most important thing if you find yourself in the situation of a damaged repository is to -stay calm and don't do anything you might regret later. - -The following point should be always considered: - -- Make a copy of you repository and try to recover from that copy. If you suspect a storage failure, - it may be even better, to make *two* copies: one to get all data out of the possibly failing storage - and another one to try the recovery process. -- Pause your regular operations on the repository or let them run on a copy. You will especially make - sure that no `forget` or `prune` is run as these command are supposed to remove data and may result - in data loss. -- Search if your issue is already known and solved. Good starting points are the restic forum and the - github issues. -- Get you some help if you are unsure what to do. Find a colleage or friend to discuss what should be done. - Also feel free to consult the restic forum. -- When using the commands below, make sure you read and understand the documentation. Some of the commands - may not be your every-day commands, so make sure you really understand what they are doing. +Make sure that you **use the latest available restic version**. It can contain +bugfixes, and improvements to simplify the repair of a repository. It might also +contain a fix for your repository problems! -2. `check` is your friend -******************************************** +1. Find out what is damaged +*************************** -Run `restic check` to find out what type of error you have. The results may be technical but can give you -a good hint what's really wrong. +The first step is always to check the repository. -Moreover, you can always run a `check` to ensure that your repair really was sucessful and your repository -is in a sane state again. -But make sure that your needed data is also still contained in your repository ;-) - -Note that `check` also prints out warning in some cases. These warnings point out that the repo may be -optimized but is still in perfect shape and does not need any troubleshooting. +.. code-block:: console -3. Index trouble -> `repair index` -******************************************** + $ restic check --read-data -A common problem with broken repostories is that the index does no longer correctly represent the contents -of your pack files. This is especially the case if some pack files got lost. -`repair index` recovers this situation and ensures that the index exactly represents the pack files. + using temporary cache in /tmp/restic-check-cache-1418935501 + repository 12345678 opened (version 2, compression level auto) + created new cache in /tmp/restic-check-cache-1418935501 + create exclusive lock for repository + load indexes + check all packs + check snapshots, trees and blobs + error for tree 7ef8ebab: + id 7ef8ebabc59aadda1a237d23ca7abac487b627a9b86508aa0194690446ff71f6 not found in repository + [0:02] 100.00% 7 / 7 snapshots + read all data + [0:05] 100.00% 25 / 25 packs + Fatal: repository contains errors -You might even need to manually remove corrupted pack files. In this case make sure, you run -`restic repair index` after. +.. note:: -Also if you encounter problems with the index files itselves, `repair index` will solve these problems -immediately. + This will download the whole repository. If retrieving data from the backend is + expensive, then omit the ``--read-data`` option. Keep a copy of the check output + as it might be necessary later on! -However, rebuilding the index does not solve every problem, e.g. lost pack files. +If the output contains warnings that the ``ciphertext verification failed`` for +some blobs in the repository, then please ask for help in the forum or our IRC +channel. These errors are often caused by hardware problems which **must** be +investigated and fixed. Otherwise, the backup will be damaged again and again. -4. Delete unneeded defect snapshots -> `forget` -******************************************** +Similarly, if a repository is repeatedly damaged, please open an `issue on Github +`_ as this could indicate a bug +somewhere. Please include the check output and additional information that might +help locate the problem. -If you encounter defect snapshots but realize you can spare them, it is often a good idea to simply -delete them using `forget`. In case that your repository remains with just sane snapshots (including -all trees and files) the next `prune` run will put your repository in a sane state. -This can be also used if you manage to create new snapshots which can replace the defect ones, see -below. +2. Backup the repository +************************ -5. No fear to `backup` again -******************************************** +Create a full copy of the repository if possible. Or at the very least make a +copy of the ``index`` and ``snapshots`` folders. This will allow you to roll back +the repository if the repair procedure fails. If your repository resides in a +cloud storage, then you can for example use `rclone `_ to +make such a copy. -There are quite some self-healing mechanisms withing the `backup` command. So it is always a good idea to -backup again and check if this did heal your repository. -If you realize that a specific file is broken in your repository and you have this file, any run of -`backup` which includes that file will be able to heal the situation. +Please disable all regular operations on the repository to prevent unexpected +changes. Especially, ``forget`` or ``prune`` must be disabled as they could +remove data unexpectedly. -Note that `backup` relies on a correct index state, so make sure your index is fine or run `repair index` -before running `backup`. +.. warning:: -6. Unreferenced tree -> `recover` -******************************************** + If you suspect hardware problems, then you *must* investigate those first. + Otherwise, the repository will soon be damaged again. -If for some reason you have unreferenced trees in your repository but you actually need them, run -`recover` it will generate a new snapshot which allows access to all trees that you have in your -repository. +Please take the time to understand what the commands described in the following +do. If you are unsure, then ask for help in the forum or our IRC channel. Search +whether your issue is already known and solved. Please take a look at the +`forum`_ and `Github issues `_. -Note that `recover` relies on a correct index state, so make sure your index is fine or run `repair index` -before running `recover`. -7. Repair defect snapshots using `repair` -******************************************** +3. Repair the index +******************* -If all other things did not help, you can repair defect snapshots with `repair`. Note that the repaired -snapshots will miss data which was referenced in the defect snapshot. +Restic relies on its index to contain correct information about what data is +stored in the repository. Thus, the first step to repair a repository is to +repair the index: + +.. code-block:: console + + $ restic repair index + + repository a14e5863 opened (version 2, compression level auto) + loading indexes... + getting pack files to read... + removing not found pack file 83ad44f59b05f6bce13376b022ac3194f24ca19e7a74926000b6e316ec6ea5a4 + rebuilding index + [0:00] 100.00% 27 / 27 packs processed + deleting obsolete index files + [0:00] 100.00% 3 / 3 files deleted + done + +This ensures that no longer existing files are removed from the index. All later +steps to repair the repository rely on a correct index. That is, you must always +repair the index first! + +Please note that it is not recommended to repair the index unless the repository +is actually damaged. + + +4. Run all backups (optional) +***************************** + +With a correct index, the ``backup`` command guarantees that newly created +snapshots can be restored successfully. It can also heal older snapshots, +if the missing data is also contained in the new snapshot. + +Therefore, it is recommended to run all your ``backup`` tasks again. In some +cases, this is enough to fully repair the repository. + + +5. Remove missing data from snapshots +************************************* + +If your repository is still missing data, then you can use the ``repair snapshots`` +command to remove all inaccessible data from the snapshots. That is, this will +result in a limited amount of data loss. Using the ``--forget`` option, the +command will automatically remove the original, damaged snapshots. + +.. code-block:: console + + $ restic repair snapshots --forget + + snapshot 6979421e of [/home/user/restic/restic] at 2022-11-02 20:59:18.617503315 +0100 CET) + file "/restic/internal/fuse/snapshots_dir.go": removed missing content + file "/restic/internal/restorer/restorer_unix_test.go": removed missing content + file "/restic/internal/walker/walker.go": removed missing content + saved new snapshot 7b094cea + removed old snapshot 6979421e + + modified 1 snapshots + +If you did not add the ``--forget`` option, then you have to manually delete all +modified snapshots using the ``forget`` command. In the example above, you'd have +to run ``restic forget 6979421e``. + + +6. Check the repository again +***************************** + +Phew, we're almost done now. To make sure that the repository has been successfully +repaired please run ``check`` again. + +.. code-block:: console + + $ restic check --read-data + + using temporary cache in /tmp/restic-check-cache-2569290785 + repository a14e5863 opened (version 2, compression level auto) + created new cache in /tmp/restic-check-cache-2569290785 + create exclusive lock for repository + load indexes + check all packs + check snapshots, trees and blobs + [0:00] 100.00% 7 / 7 snapshots + read all data + [0:00] 100.00% 25 / 25 packs + no errors were found + +If the ``check`` command did not complete with ``no errors were found``, then +the repository is still damaged. At this point, please ask for help at the +`forum`_ or our IRC channel ``#restic`` on ``irc.libera.chat``. From 5aa37acdaa39f03b28dfda6a5cb13339b41eddf7 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 28 Dec 2022 15:32:00 +0100 Subject: [PATCH 115/127] repair snapshots: cleanup command help --- cmd/restic/cmd_repair_snapshots.go | 33 ++++++++++++++---------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index e50d34b9d..5e9ec4130 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -15,28 +15,25 @@ var cmdRepairSnapshots = &cobra.Command{ Use: "snapshots [flags] [snapshot ID] [...]", Short: "Repair snapshots", Long: ` -The "repair snapshots" command allows to repair broken snapshots. -It scans the given snapshots and generates new ones where -damaged tress and file contents are removed. -If the broken snapshots are deleted, a prune run will -be able to refit the repository. +The "repair snapshots" command repairs broken snapshots. It scans the given +snapshots and generates new ones with damaged directories and file contents +removed. If the broken snapshots are deleted, a prune run will be able to +clean up the repository. -The command depends on a good state of the index, so if -there are inaccurancies in the index, make sure to run -"repair index" before! +The command depends on a correct index, thus make sure to run "repair index" +first! -WARNING: -======== -Repairing and deleting broken snapshots causes data loss! -It will remove broken dirs and modify broken files in -the modified snapshots. +WARNING +======= -If the contents of directories and files are still available, -the better option is to redo a backup which in that case is -able to "heal" already present snapshots. -Only use this command if you need to recover an old and -broken snapshot! +Repairing and deleting broken snapshots causes data loss! It will remove broken +directories and modify broken files in the modified snapshots. + +If the contents of directories and files are still available, the better option +is to run "backup" which in that case is able to heal existing snapshots. Only +use the "repair snapshots" command if you need to recover an old and broken +snapshot! EXIT STATUS =========== From e71367e6b90ae44477dbb53bbbf4698746376375 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 28 Dec 2022 15:40:59 +0100 Subject: [PATCH 116/127] repair snapshots: update changelog --- changelog/unreleased/issue-1759 | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/changelog/unreleased/issue-1759 b/changelog/unreleased/issue-1759 index 0b0d28aa5..1b698f845 100644 --- a/changelog/unreleased/issue-1759 +++ b/changelog/unreleased/issue-1759 @@ -1,14 +1,18 @@ -Enhancement: Add new command repair +Enhancement: Add `repair index` and `repair snapshots` commands -We've added a new command repair which allows to repair snapshots even if needed -parts of it are not accessable in the repository. Note that using this command -can lead to data loss! +The `rebuild-index` command has been renamed to `repair index`. The old name +will still work, but is deprecated. -Some corrupted repositories were reported in several issues and so far restic -lacked a possibility to accept data loss but clean those up such that the -repository returns to a sane state. This possibility was now added. +When a snapshot was damaged, the only option up to now was to completely forget +the snapshot, even if only some unimportant file was damaged. + +We've added a `repair snapshots` command, which can repair snapshots by removing +damaged directories and missing files contents. Note that using this command +can lead to data loss! Please see the "Troubleshooting" section in the documentation +for more details. https://github.com/restic/restic/issues/1759 +https://github.com/restic/restic/issues/1714 https://github.com/restic/restic/issues/1798 https://github.com/restic/restic/issues/2334 https://github.com/restic/restic/pull/2876 From c0627dc80d66397747d6f5aa7210649b9906e5fc Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 1 May 2023 17:18:19 +0200 Subject: [PATCH 117/127] check: Fix flaky TestCheckerModifiedData The test had a 4% chance of not modified the data read from the repository, in which case the test would fail. Change the data manipulation to just modified each read operation. --- internal/checker/checker_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/checker/checker_test.go b/internal/checker/checker_test.go index 775484652..6405ecfbd 100644 --- a/internal/checker/checker_test.go +++ b/internal/checker/checker_test.go @@ -331,10 +331,6 @@ func (erd errorReadCloser) Read(p []byte) (int, error) { // induceError flips a bit in the slice. func induceError(data []byte) { - if rand.Float32() < 0.2 { - return - } - pos := rand.Intn(len(data)) data[pos] ^= 1 } From 88a7231217ce3118eb3ffff5ee1f59f2f8c6ccc7 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 1 May 2023 17:24:13 +0200 Subject: [PATCH 118/127] report snapshot id if loading failed --- internal/restic/snapshot.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/restic/snapshot.go b/internal/restic/snapshot.go index 58d863526..1f6e4534b 100644 --- a/internal/restic/snapshot.go +++ b/internal/restic/snapshot.go @@ -61,7 +61,7 @@ func LoadSnapshot(ctx context.Context, loader LoaderUnpacked, id ID) (*Snapshot, sn := &Snapshot{id: &id} err := LoadJSONUnpacked(ctx, loader, SnapshotFile, id, sn) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to load snapshot %v: %w", id.Str(), err) } return sn, nil From 1cb11ad8add274cfb727bf41c2933903848d5b9a Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 1 May 2023 18:03:17 +0200 Subject: [PATCH 119/127] mount: enable debug logging for the flaky TestMount test The test case fails from time to time with an Input/Output error while trying to access the snapshots directory. --- cmd/restic/integration_fuse_test.go | 6 ++++++ internal/debug/testing.go | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 internal/debug/testing.go diff --git a/cmd/restic/integration_fuse_test.go b/cmd/restic/integration_fuse_test.go index a99064b8f..b69886024 100644 --- a/cmd/restic/integration_fuse_test.go +++ b/cmd/restic/integration_fuse_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -159,6 +160,11 @@ func TestMount(t *testing.T) { t.Skip("Skipping fuse tests") } + debugEnabled := debug.TestLogToStderr(t) + if debugEnabled { + defer debug.TestDisableLog(t) + } + env, cleanup := withTestEnvironment(t) // must list snapshots more than once env.gopts.backendTestHook = nil diff --git a/internal/debug/testing.go b/internal/debug/testing.go new file mode 100644 index 000000000..c9ceae0ea --- /dev/null +++ b/internal/debug/testing.go @@ -0,0 +1,23 @@ +package debug + +import ( + "log" + "os" + "testing" +) + +// TestLogToStderr configures debug to log to stderr if not the debug log is +// not already configured and returns whether logging was enabled. +func TestLogToStderr(t testing.TB) bool { + if opts.isEnabled { + return false + } + opts.logger = log.New(os.Stderr, "", log.LstdFlags) + opts.isEnabled = true + return true +} + +func TestDisableLog(t testing.TB) { + opts.logger = nil + opts.isEnabled = false +} From 78e5aa6d30402a6bead64515dd224963adc38dff Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 4 May 2023 23:00:46 +0200 Subject: [PATCH 120/127] repair snapshots: add basic tests --- .../integration_repair_snapshots_test.go | 135 ++++++++++++++++++ cmd/restic/integration_test.go | 23 ++- 2 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 cmd/restic/integration_repair_snapshots_test.go diff --git a/cmd/restic/integration_repair_snapshots_test.go b/cmd/restic/integration_repair_snapshots_test.go new file mode 100644 index 000000000..04ef6ad1d --- /dev/null +++ b/cmd/restic/integration_repair_snapshots_test.go @@ -0,0 +1,135 @@ +package main + +import ( + "context" + "hash/fnv" + "io" + "math/rand" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" +) + +func testRunRepairSnapshot(t testing.TB, gopts GlobalOptions, forget bool) { + opts := RepairOptions{ + Forget: forget, + } + + rtest.OK(t, runRepairSnapshots(context.TODO(), gopts, opts, nil)) +} + +func createRandomFile(t testing.TB, env *testEnvironment, path string, size int) { + fn := filepath.Join(env.testdata, path) + rtest.OK(t, os.MkdirAll(filepath.Dir(fn), 0o755)) + + h := fnv.New64() + _, err := h.Write([]byte(path)) + rtest.OK(t, err) + r := rand.New(rand.NewSource(int64(h.Sum64()))) + + f, err := os.OpenFile(fn, os.O_CREATE|os.O_RDWR, 0o644) + rtest.OK(t, err) + _, err = io.Copy(f, io.LimitReader(r, int64(size))) + rtest.OK(t, err) + rtest.OK(t, f.Close()) +} + +func TestRepairSnapshotsWithLostData(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + + createRandomFile(t, env, "foo/bar/file", 512*1024) + testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts) + testListSnapshots(t, env.gopts, 1) + // damage repository + removePacksExcept(env.gopts, t, restic.NewIDSet(), false) + + createRandomFile(t, env, "foo/bar/file2", 256*1024) + testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts) + snapshotIDs := testListSnapshots(t, env.gopts, 2) + testRunCheckMustFail(t, env.gopts) + + // repair but keep broken snapshots + testRunRebuildIndex(t, env.gopts) + testRunRepairSnapshot(t, env.gopts, false) + testListSnapshots(t, env.gopts, 4) + testRunCheckMustFail(t, env.gopts) + + // repository must be ok after removing the broken snapshots + testRunForget(t, env.gopts, snapshotIDs[0].String(), snapshotIDs[1].String()) + testListSnapshots(t, env.gopts, 2) + _, err := testRunCheckOutput(env.gopts) + rtest.OK(t, err) +} + +func TestRepairSnapshotsWithLostTree(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + + createRandomFile(t, env, "foo/bar/file", 12345) + testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts) + oldSnapshot := testListSnapshots(t, env.gopts, 1) + oldPacks := testRunList(t, "packs", env.gopts) + + // keep foo/bar unchanged + createRandomFile(t, env, "foo/bar2", 1024) + testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts) + testListSnapshots(t, env.gopts, 2) + + // remove tree for foo/bar and the now completely broken first snapshot + removePacks(env.gopts, t, restic.NewIDSet(oldPacks...)) + testRunForget(t, env.gopts, oldSnapshot[0].String()) + testRunCheckMustFail(t, env.gopts) + + // repair + testRunRebuildIndex(t, env.gopts) + testRunRepairSnapshot(t, env.gopts, true) + testListSnapshots(t, env.gopts, 1) + _, err := testRunCheckOutput(env.gopts) + rtest.OK(t, err) +} + +func TestRepairSnapshotsWithLostRootTree(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + + createRandomFile(t, env, "foo/bar/file", 12345) + testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts) + testListSnapshots(t, env.gopts, 1) + oldPacks := testRunList(t, "packs", env.gopts) + + // remove all trees + removePacks(env.gopts, t, restic.NewIDSet(oldPacks...)) + testRunCheckMustFail(t, env.gopts) + + // repair + testRunRebuildIndex(t, env.gopts) + testRunRepairSnapshot(t, env.gopts, true) + testListSnapshots(t, env.gopts, 0) + _, err := testRunCheckOutput(env.gopts) + rtest.OK(t, err) +} + +func TestRepairSnapshotsIntact(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + testSetupBackupData(t, env) + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{}, env.gopts) + oldSnapshotIDs := testListSnapshots(t, env.gopts, 1) + + // use an exclude that will not exclude anything + testRunRepairSnapshot(t, env.gopts, false) + snapshotIDs := testListSnapshots(t, env.gopts, 1) + rtest.Assert(t, reflect.DeepEqual(oldSnapshotIDs, snapshotIDs), "unexpected snapshot id mismatch %v vs. %v", oldSnapshotIDs, snapshotIDs) + testRunCheck(t, env.gopts) +} diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 42fd26d6b..211089253 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -100,6 +100,13 @@ func testRunList(t testing.TB, tpe string, opts GlobalOptions) restic.IDs { return parseIDsFromReader(t, buf) } +func testListSnapshots(t testing.TB, opts GlobalOptions, expected int) restic.IDs { + t.Helper() + snapshotIDs := testRunList(t, "snapshots", opts) + rtest.Assert(t, len(snapshotIDs) == expected, "expected %v snapshot, got %v", expected, snapshotIDs) + return snapshotIDs +} + func testRunRestore(t testing.TB, opts GlobalOptions, dir string, snapshotID restic.ID) { testRunRestoreExcludes(t, opts, dir, snapshotID, nil) } @@ -164,6 +171,11 @@ func testRunCheckOutput(gopts GlobalOptions) (string, error) { return buf.String(), err } +func testRunCheckMustFail(t testing.TB, gopts GlobalOptions) { + _, err := testRunCheckOutput(gopts) + rtest.Assert(t, err != nil, "expected non nil error after check of damaged repository") +} + func testRunDiffOutput(gopts GlobalOptions, firstSnapshotID string, secondSnapshotID string) (string, error) { buf := bytes.NewBuffer(nil) @@ -486,7 +498,16 @@ func TestBackupNonExistingFile(t *testing.T) { testRunBackup(t, "", dirs, opts, env.gopts) } -func removePacksExcept(gopts GlobalOptions, t *testing.T, keep restic.IDSet, removeTreePacks bool) { +func removePacks(gopts GlobalOptions, t testing.TB, remove restic.IDSet) { + r, err := OpenRepository(context.TODO(), gopts) + rtest.OK(t, err) + + for id := range remove { + rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()})) + } +} + +func removePacksExcept(gopts GlobalOptions, t testing.TB, keep restic.IDSet, removeTreePacks bool) { r, err := OpenRepository(context.TODO(), gopts) rtest.OK(t, err) From bb40e49e75175e4c71df4390facc51261b1ebc60 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 1 May 2023 18:13:10 +0200 Subject: [PATCH 121/127] ui/termstatus: Fix truncation of status output The last line was not truncated as expected --- internal/ui/termstatus/status.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/ui/termstatus/status.go b/internal/ui/termstatus/status.go index 46bfb2868..6939f5e28 100644 --- a/internal/ui/termstatus/status.go +++ b/internal/ui/termstatus/status.go @@ -359,8 +359,9 @@ func (t *Terminal) SetStatus(lines []string) { line = Truncate(line, width-2) } if i < len(lines)-1 { // Last line gets no line break. - lines[i] = line + "\n" + line += "\n" } + lines[i] = line } select { From 6d10c655a05824cfc3837fda034758c23776d701 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 1 May 2023 18:21:08 +0200 Subject: [PATCH 122/127] termstatus: test status line sanitization --- changelog/unreleased/pull-4318 | 8 ++++++++ internal/ui/termstatus/status.go | 27 ++++++++++++++++----------- internal/ui/termstatus/status_test.go | 24 ++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 changelog/unreleased/pull-4318 diff --git a/changelog/unreleased/pull-4318 b/changelog/unreleased/pull-4318 new file mode 100644 index 000000000..198b972d3 --- /dev/null +++ b/changelog/unreleased/pull-4318 @@ -0,0 +1,8 @@ +Bugfix: Correctly clean up status bar output of the `backup` command + +Due to a regression in restic 0.15.2, the status bar of the `backup` command +could leave some output behind. This happened if filenames were printed that +are wider than the current terminal width. This has been fixed. + +https://github.com/restic/restic/issues/4319 +https://github.com/restic/restic/pull/4318 diff --git a/internal/ui/termstatus/status.go b/internal/ui/termstatus/status.go index 6939f5e28..5b310ec80 100644 --- a/internal/ui/termstatus/status.go +++ b/internal/ui/termstatus/status.go @@ -334,6 +334,21 @@ func wideRune(s string) (wide bool, utfsize uint) { return wide, uint(size) } +func sanitizeLines(lines []string, width int) []string { + // Sanitize lines and truncate them if they're too long. + for i, line := range lines { + line = Quote(line) + if width > 0 { + line = Truncate(line, width-2) + } + if i < len(lines)-1 { // Last line gets no line break. + line += "\n" + } + lines[i] = line + } + return lines +} + // SetStatus updates the status lines. // The lines should not contain newlines; this method adds them. func (t *Terminal) SetStatus(lines []string) { @@ -352,17 +367,7 @@ func (t *Terminal) SetStatus(lines []string) { } } - // Sanitize lines and truncate them if they're too long. - for i, line := range lines { - line = Quote(line) - if width > 0 { - line = Truncate(line, width-2) - } - if i < len(lines)-1 { // Last line gets no line break. - line += "\n" - } - lines[i] = line - } + sanitizeLines(lines, width) select { case t.status <- status{lines: lines}: diff --git a/internal/ui/termstatus/status_test.go b/internal/ui/termstatus/status_test.go index 9f5e15cb1..94eb734a7 100644 --- a/internal/ui/termstatus/status_test.go +++ b/internal/ui/termstatus/status_test.go @@ -1,6 +1,7 @@ package termstatus import ( + "reflect" "strconv" "testing" @@ -91,3 +92,26 @@ func BenchmarkTruncateUnicode(b *testing.B) { benchmarkTruncate(b, s, w-1) } + +func TestSanitizeLines(t *testing.T) { + var tests = []struct { + input []string + width int + output []string + }{ + {[]string{""}, 80, []string{""}}, + {[]string{"too long test line"}, 10, []string{"too long"}}, + {[]string{"too long test line", "text"}, 10, []string{"too long\n", "text"}}, + {[]string{"too long test line", "second long test line"}, 10, []string{"too long\n", "second l"}}, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + out := sanitizeLines(test.input, test.width) + if !reflect.DeepEqual(out, test.output) { + t.Fatalf("wrong output for input %v, width %d: want %q, got %q", + test.input, test.width, test.output, out) + } + }) + } +} From 65c5e511a16bbcfc1e744ea7e05fbf72bbff1018 Mon Sep 17 00:00:00 2001 From: greatroar <61184462+greatroar@users.noreply.github.com> Date: Fri, 5 May 2023 11:10:02 +0200 Subject: [PATCH 123/127] ui/termstatus: Add test for Terminal.SetStatus --- internal/ui/termstatus/status_test.go | 50 +++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/internal/ui/termstatus/status_test.go b/internal/ui/termstatus/status_test.go index 94eb734a7..b59063076 100644 --- a/internal/ui/termstatus/status_test.go +++ b/internal/ui/termstatus/status_test.go @@ -1,13 +1,54 @@ package termstatus import ( - "reflect" + "bytes" + "context" + "fmt" + "io" "strconv" "testing" rtest "github.com/restic/restic/internal/test" ) +func TestSetStatus(t *testing.T) { + var buf bytes.Buffer + term := New(&buf, io.Discard, false) + + term.canUpdateStatus = true + term.fd = ^uintptr(0) + term.clearCurrentLine = posixClearCurrentLine + term.moveCursorUp = posixMoveCursorUp + + ctx, cancel := context.WithCancel(context.Background()) + go term.Run(ctx) + + const ( + clear = posixControlClearLine + home = posixControlMoveCursorHome + up = posixControlMoveCursorUp + ) + + term.SetStatus([]string{"first"}) + exp := home + clear + "first" + home + + term.SetStatus([]string{"foo", "bar", "baz"}) + exp += home + clear + "foo\n" + home + clear + "bar\n" + + home + clear + "baz" + home + up + up + + term.SetStatus([]string{"quux", "needs\nquote"}) + exp += home + clear + "quux\n" + + home + clear + "\"needs\\nquote\"\n" + + home + clear + home + up + up // Third line implicit. + + cancel() + exp += home + clear + "\n" + home + clear + "\n" + + home + up + up // Status cleared. + + <-term.closed + rtest.Equals(t, exp, buf.String()) +} + func TestQuote(t *testing.T) { for _, c := range []struct { in string @@ -106,12 +147,9 @@ func TestSanitizeLines(t *testing.T) { } for _, test := range tests { - t.Run("", func(t *testing.T) { + t.Run(fmt.Sprintf("%s %d", test.input, test.width), func(t *testing.T) { out := sanitizeLines(test.input, test.width) - if !reflect.DeepEqual(out, test.output) { - t.Fatalf("wrong output for input %v, width %d: want %q, got %q", - test.input, test.width, test.output, out) - } + rtest.Equals(t, test.output, out) }) } } From d05f6211d151783cb2f1a26d05e5965ad78a89db Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 23 Apr 2023 12:16:54 +0200 Subject: [PATCH 124/127] lock: Do not limit backend concurrency for lock files restic must be able to refresh lock files in time. However, large uploads over slow connections can cause the lock refresh to be stuck behind the large uploads and thus time out. --- changelog/unreleased/pull-4304 | 5 ++++ internal/backend/sema/backend.go | 23 ++++++++++++------- internal/backend/sema/backend_test.go | 33 +++++++++++++++++++++------ 3 files changed, 46 insertions(+), 15 deletions(-) create mode 100644 changelog/unreleased/pull-4304 diff --git a/changelog/unreleased/pull-4304 b/changelog/unreleased/pull-4304 new file mode 100644 index 000000000..ca3c7a8db --- /dev/null +++ b/changelog/unreleased/pull-4304 @@ -0,0 +1,5 @@ +Bugfix: Avoid lock refresh issues with slow network connections + +On network connections with a low upload speed, restic could often fail backups and other operations with `Fatal: failed to refresh lock in time`. We've reworked the lock refresh to avoid this error. + +https://github.com/restic/restic/pull/4304 diff --git a/internal/backend/sema/backend.go b/internal/backend/sema/backend.go index fc4a9dde5..dd4859ed1 100644 --- a/internal/backend/sema/backend.go +++ b/internal/backend/sema/backend.go @@ -31,14 +31,24 @@ func NewBackend(be restic.Backend) restic.Backend { } } +// typeDependentLimit acquire a token unless the FileType is a lock file. The returned function +// must be called to release the token. +func (be *connectionLimitedBackend) typeDependentLimit(t restic.FileType) func() { + // allow concurrent lock file operations to ensure that the lock refresh is always possible + if t == restic.LockFile { + return func() {} + } + be.sem.GetToken() + return be.sem.ReleaseToken +} + // Save adds new Data to the backend. func (be *connectionLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { if err := h.Valid(); err != nil { return backoff.Permanent(err) } - be.sem.GetToken() - defer be.sem.ReleaseToken() + defer be.typeDependentLimit(h.Type)() return be.Backend.Save(ctx, h, rd) } @@ -56,8 +66,7 @@ func (be *connectionLimitedBackend) Load(ctx context.Context, h restic.Handle, l return backoff.Permanent(errors.Errorf("invalid length %d", length)) } - be.sem.GetToken() - defer be.sem.ReleaseToken() + defer be.typeDependentLimit(h.Type)() return be.Backend.Load(ctx, h, length, offset, fn) } @@ -68,8 +77,7 @@ func (be *connectionLimitedBackend) Stat(ctx context.Context, h restic.Handle) ( return restic.FileInfo{}, backoff.Permanent(err) } - be.sem.GetToken() - defer be.sem.ReleaseToken() + defer be.typeDependentLimit(h.Type)() return be.Backend.Stat(ctx, h) } @@ -80,8 +88,7 @@ func (be *connectionLimitedBackend) Remove(ctx context.Context, h restic.Handle) return backoff.Permanent(err) } - be.sem.GetToken() - defer be.sem.ReleaseToken() + defer be.typeDependentLimit(h.Type)() return be.Backend.Remove(ctx, h) } diff --git a/internal/backend/sema/backend_test.go b/internal/backend/sema/backend_test.go index db9559840..dc599b7f8 100644 --- a/internal/backend/sema/backend_test.go +++ b/internal/backend/sema/backend_test.go @@ -88,7 +88,7 @@ func countingBlocker() (func(), func(int) int) { unblock := func(expected int) int { // give goroutines enough time to block var blocked int64 - for i := 0; i < 100 && blocked != int64(expected); i++ { + for i := 0; i < 100 && blocked < int64(expected); i++ { time.Sleep(100 * time.Microsecond) blocked = atomic.LoadInt64(&ctr) } @@ -99,8 +99,9 @@ func countingBlocker() (func(), func(int) int) { return wait, unblock } -func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(be restic.Backend) func() error, unblock func(int) int) { +func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(be restic.Backend) func() error, unblock func(int) int, isUnlimited bool) { expectBlocked := int(2) + workerCount := expectBlocked + 1 m := mock.NewBackend() setup(m) @@ -108,10 +109,13 @@ func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(b be := sema.NewBackend(m) var wg errgroup.Group - for i := 0; i < int(expectBlocked+1); i++ { + for i := 0; i < workerCount; i++ { wg.Go(handler(be)) } + if isUnlimited { + expectBlocked = workerCount + } blocked := unblock(expectBlocked) test.Assert(t, blocked == expectBlocked, "Unexpected number of goroutines blocked: %v", blocked) test.OK(t, wg.Wait()) @@ -129,7 +133,7 @@ func TestConcurrencyLimitSave(t *testing.T) { h := restic.Handle{Type: restic.PackFile, Name: "foobar"} return be.Save(context.TODO(), h, nil) } - }, unblock) + }, unblock, false) } func TestConcurrencyLimitLoad(t *testing.T) { @@ -145,7 +149,7 @@ func TestConcurrencyLimitLoad(t *testing.T) { nilCb := func(rd io.Reader) error { return nil } return be.Load(context.TODO(), h, 10, 0, nilCb) } - }, unblock) + }, unblock, false) } func TestConcurrencyLimitStat(t *testing.T) { @@ -161,7 +165,7 @@ func TestConcurrencyLimitStat(t *testing.T) { _, err := be.Stat(context.TODO(), h) return err } - }, unblock) + }, unblock, false) } func TestConcurrencyLimitDelete(t *testing.T) { @@ -176,5 +180,20 @@ func TestConcurrencyLimitDelete(t *testing.T) { h := restic.Handle{Type: restic.PackFile, Name: "foobar"} return be.Remove(context.TODO(), h) } - }, unblock) + }, unblock, false) +} + +func TestConcurrencyUnlimitedLockSave(t *testing.T) { + wait, unblock := countingBlocker() + concurrencyTester(t, func(m *mock.Backend) { + m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + wait() + return nil + } + }, func(be restic.Backend) func() error { + return func() error { + h := restic.Handle{Type: restic.LockFile, Name: "foobar"} + return be.Save(context.TODO(), h, nil) + } + }, unblock, true) } From e77002f84112b28ec89d89d899a68a4e062667c0 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 1 May 2023 12:02:50 +0200 Subject: [PATCH 125/127] restore: correctly count hardlinks in progress bar For hardlinked files, only the first instance of that file increases the amount of bytes to restore. All later instances only increase the file count but not the restore size. --- internal/restorer/restorer.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 289883ed0..88eeee658 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -257,21 +257,28 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { return nil } - if res.progress != nil { - res.progress.AddFile(node.Size) - } - if node.Size == 0 { + if res.progress != nil { + res.progress.AddFile(node.Size) + } return nil // deal with empty files later } if node.Links > 1 { if idx.Has(node.Inode, node.DeviceID) { + if res.progress != nil { + // a hardlinked file does not increase the restore size + res.progress.AddFile(0) + } return nil } idx.Add(node.Inode, node.DeviceID, location) } + if res.progress != nil { + res.progress.AddFile(node.Size) + } + filerestorer.addFile(location, node.Content, int64(node.Size)) return nil From 23a122a9013641466f631e8c79982d1be5988efc Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 1 May 2023 12:05:48 +0200 Subject: [PATCH 126/127] restore: count files in the same way as the stats command --- internal/restorer/restorer.go | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 88eeee658..4acd45f95 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -166,12 +166,14 @@ func (res *Restorer) restoreNodeTo(ctx context.Context, node *restic.Node, targe err := node.CreateAt(ctx, target, res.repo) if err != nil { debug.Log("node.CreateAt(%s) error %v", target, err) - } - if err == nil { - err = res.restoreNodeMetadataTo(node, target, location) + return err } - return err + if res.progress != nil { + res.progress.AddProgress(location, 0, 0) + } + + return res.restoreNodeMetadataTo(node, target, location) } func (res *Restorer) restoreNodeMetadataTo(node *restic.Node, target, location string) error { @@ -239,6 +241,9 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { _, err = res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{ enterDir: func(node *restic.Node, target, location string) error { debug.Log("first pass, enterDir: mkdir %q, leaveDir should restore metadata", location) + if res.progress != nil { + res.progress.AddFile(0) + } // create dir with default permissions // #leaveDir restores dir metadata after visiting all children return fs.MkdirAll(target, 0700) @@ -254,6 +259,9 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { } if node.Type != "file" { + if res.progress != nil { + res.progress.AddFile(0) + } return nil } @@ -317,7 +325,13 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { return res.restoreNodeMetadataTo(node, target, location) }, - leaveDir: res.restoreNodeMetadataTo, + leaveDir: func(node *restic.Node, target, location string) error { + err := res.restoreNodeMetadataTo(node, target, location) + if err == nil && res.progress != nil { + res.progress.AddProgress(location, 0, 0) + } + return err + }, }) return err } From 19ebc1b786bf66f9c7e1c6a1e76c7eb0f0eac21c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 8 May 2023 20:49:41 +0200 Subject: [PATCH 127/127] restore: Add basic test for progress bar accounting of hardlinks --- internal/restorer/restorer_unix_test.go | 55 +++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/internal/restorer/restorer_unix_test.go b/internal/restorer/restorer_unix_test.go index e9c521e36..4c5f2a5b8 100644 --- a/internal/restorer/restorer_unix_test.go +++ b/internal/restorer/restorer_unix_test.go @@ -9,10 +9,12 @@ import ( "path/filepath" "syscall" "testing" + "time" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" + restoreui "github.com/restic/restic/internal/ui/restore" ) func TestRestorerRestoreEmptyHardlinkedFileds(t *testing.T) { @@ -66,3 +68,56 @@ func getBlockCount(t *testing.T, filename string) int64 { } return st.Blocks } + +type printerMock struct { + filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64 +} + +func (p *printerMock) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { +} +func (p *printerMock) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { + p.filesFinished = filesFinished + p.filesTotal = filesTotal + p.allBytesWritten = allBytesWritten + p.allBytesTotal = allBytesTotal +} + +func TestRestorerProgressBar(t *testing.T) { + repo := repository.TestRepository(t) + + sn, _ := saveSnapshot(t, repo, Snapshot{ + Nodes: map[string]Node{ + "dirtest": Dir{ + Nodes: map[string]Node{ + "file1": File{Links: 2, Inode: 1, Data: "foo"}, + "file2": File{Links: 2, Inode: 1, Data: "foo"}, + }, + }, + "file2": File{Links: 1, Inode: 2, Data: "example"}, + }, + }) + + mock := &printerMock{} + progress := restoreui.NewProgress(mock, 0) + res := NewRestorer(context.TODO(), repo, sn, false, progress) + res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { + return true, true + } + + tempdir := rtest.TempDir(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) + progress.Finish() + + const filesFinished = 4 + const filesTotal = filesFinished + const allBytesWritten = 10 + const allBytesTotal = allBytesWritten + rtest.Assert(t, mock.filesFinished == filesFinished, "filesFinished: expected %v, got %v", filesFinished, mock.filesFinished) + rtest.Assert(t, mock.filesTotal == filesTotal, "filesTotal: expected %v, got %v", filesTotal, mock.filesTotal) + rtest.Assert(t, mock.allBytesWritten == allBytesWritten, "allBytesWritten: expected %v, got %v", allBytesWritten, mock.allBytesWritten) + rtest.Assert(t, mock.allBytesTotal == allBytesTotal, "allBytesTotal: expected %v, got %v", allBytesTotal, mock.allBytesTotal) +}