From 146587416ff40f4a08f98c4bd578b15c4e975cd8 Mon Sep 17 00:00:00 2001 From: Kjetil Torgrim Homme Date: Mon, 30 Dec 2024 22:06:27 +0100 Subject: [PATCH 1/7] Add snapshot filtering --older-than and --newer-than This would solve issue #5136 --- cmd/restic/find.go | 2 ++ internal/restic/duration.go | 13 +++++++ internal/restic/duration_test.go | 16 +++++++++ internal/restic/snapshot.go | 16 +++++++++ internal/restic/snapshot_find.go | 8 +++-- internal/restic/snapshot_find_test.go | 51 +++++++++++++++++++++++++++ 6 files changed, 104 insertions(+), 2 deletions(-) diff --git a/cmd/restic/find.go b/cmd/restic/find.go index faf7024e1..407fcf5e4 100644 --- a/cmd/restic/find.go +++ b/cmd/restic/find.go @@ -17,6 +17,8 @@ func initMultiSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter, } flags.StringArrayVarP(&filt.Hosts, "host", hostShorthand, nil, "only consider snapshots for this `host` (can be specified multiple times) (default: $RESTIC_HOST)") flags.Var(&filt.Tags, "tag", "only consider snapshots including `tag[,tag,...]` (can be specified multiple times)") + flags.Var(&filt.OlderThan, "older-than", "only consider snapshots made more than `duration` time ago") + flags.Var(&filt.NewerThan, "newer-than", "only consider snapshots made less than `duration` time ago") flags.StringArrayVar(&filt.Paths, "path", nil, "only consider snapshots including this (absolute) `path` (can be specified multiple times, snapshots must include all specified paths)") // set default based on env if set diff --git a/internal/restic/duration.go b/internal/restic/duration.go index 831971fe0..ed5acf24e 100644 --- a/internal/restic/duration.go +++ b/internal/restic/duration.go @@ -4,6 +4,7 @@ import ( "fmt" "strconv" "strings" + "time" "unicode" "github.com/restic/restic/internal/errors" @@ -126,6 +127,18 @@ func (d *Duration) Set(s string) error { return nil } +// Subtract duration from t to return past point in time. If t is not given, use time.Now(). +func (d Duration) PastTime(t ...time.Time) time.Time { + var reftime time.Time + if t == nil { + reftime = time.Now() + } else { + reftime = t[0] + } + + return reftime.AddDate(-d.Years, -d.Months, -d.Days).Add(-time.Duration(d.Hours) * time.Hour) +} + // Type returns the type of Duration, usable within github.com/spf13/pflag and // in help texts. func (d Duration) Type() string { diff --git a/internal/restic/duration_test.go b/internal/restic/duration_test.go index f03aa5553..60a618f08 100644 --- a/internal/restic/duration_test.go +++ b/internal/restic/duration_test.go @@ -2,6 +2,7 @@ package restic import ( "testing" + "time" "github.com/google/go-cmp/cmp" ) @@ -104,3 +105,18 @@ func TestParseDuration(t *testing.T) { }) } } + +func TestPastTime(t *testing.T) { + t.Run("", func(t *testing.T) { + d, err := ParseDuration("1y2m3d4h") + if err != nil { + t.Fatal(err) + } + reftime, err := time.Parse(time.DateTime, "1999-12-30 15:16:17") + expected := "1998-10-27 11:16:17" + result := d.PastTime(reftime).Format(time.DateTime) + if result != expected { + t.Errorf("unexpected return of PastTime, wanted %q, got %q", expected, result) + } + }) +} diff --git a/internal/restic/snapshot.go b/internal/restic/snapshot.go index f9cdf4daf..247b7c5f0 100644 --- a/internal/restic/snapshot.go +++ b/internal/restic/snapshot.go @@ -248,6 +248,22 @@ func (sn *Snapshot) HasHostname(hostnames []string) bool { return false } +// HasTimeBetween returns false if either +// - start is given and Time is before start, or +// - stop is given and Time is after stop +// Otherwise return true +// start is the value closest in time, ie. the shortest duration value. +func (sn *Snapshot) HasTimeBetween(start Duration, stop Duration) bool { + if !start.Zero() && sn.Time.After(start.PastTime()) { + return false + } + if !stop.Zero() && sn.Time.Before(stop.PastTime()) { + return false + } + + return true +} + // Snapshots is a list of snapshots. type Snapshots []*Snapshot diff --git a/internal/restic/snapshot_find.go b/internal/restic/snapshot_find.go index 6eb51b237..39d8765d4 100644 --- a/internal/restic/snapshot_find.go +++ b/internal/restic/snapshot_find.go @@ -20,8 +20,12 @@ type SnapshotFilter struct { Hosts []string Tags TagLists Paths []string - // Match snapshots from before this timestamp. Zero for no limit. + // Match snapshots from before this timestamp. Zero for no limit. Only used to find parent. TimestampLimit time.Time + // Match snapshots from before this timestamp. Zero for no limit. + NewerThan Duration + // Match snapshots from after this timestamp. Zero for no limit. + OlderThan Duration } func (f *SnapshotFilter) Empty() bool { @@ -29,7 +33,7 @@ func (f *SnapshotFilter) Empty() bool { } func (f *SnapshotFilter) matches(sn *Snapshot) bool { - return sn.HasHostname(f.Hosts) && sn.HasTagList(f.Tags) && sn.HasPaths(f.Paths) + return sn.HasHostname(f.Hosts) && sn.HasTagList(f.Tags) && sn.HasPaths(f.Paths) && sn.HasTimeBetween(f.OlderThan, f.NewerThan) } // findLatest finds the latest snapshot with optional target/directory, diff --git a/internal/restic/snapshot_find_test.go b/internal/restic/snapshot_find_test.go index 84bffd694..83c2e5c91 100644 --- a/internal/restic/snapshot_find_test.go +++ b/internal/restic/snapshot_find_test.go @@ -3,6 +3,7 @@ package restic_test import ( "context" "testing" + "time" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -89,3 +90,53 @@ func TestFindAllSubpathError(t *testing.T) { })) test.Assert(t, count == 2, "unexpected number of subfolder errors: %v, wanted %v", count, 2) } + +func TestFindAllNewerThan(t *testing.T) { + repo := repository.TestRepository(t) + now := time.Now() + + oneDay := time.Duration(24) * time.Hour + restic.TestCreateSnapshot(t, repo, now.Add(-14*oneDay), 1) + desiredSnapshot := restic.TestCreateSnapshot(t, repo, now.Add(-4*oneDay), 1) + + var found restic.Snapshot + count := 0 + test.OK(t, (&restic.SnapshotFilter{ + NewerThan: restic.Duration{Days: 5}, + }).FindAll(context.TODO(), repo, repo, nil, + func(id string, sn *restic.Snapshot, err error) error { + if err == nil { + found = *sn + count++ + } + return nil + })) + test.Assert(t, count == 1, "unexpected number of snapshots: %v, wanted %v", count, 1) + test.Assert(t, desiredSnapshot.ID().Equal(*found.ID()), "unexpected snapshot found: %s, wanted %s", desiredSnapshot, found) +} + +func TestFindAllWithin(t *testing.T) { + repo := repository.TestRepository(t) + now := time.Now() + + oneDay := time.Duration(24) * time.Hour + restic.TestCreateSnapshot(t, repo, now.Add(-14*oneDay), 1) + restic.TestCreateSnapshot(t, repo, now.Add(-1*oneDay), 1) + desiredSnapshot := restic.TestCreateSnapshot(t, repo, now.Add(-4*oneDay), 1) + + var found restic.Snapshot + count := 0 + test.OK(t, (&restic.SnapshotFilter{ + NewerThan: restic.Duration{Days: 5}, + OlderThan: restic.Duration{Days: 2}, + }).FindAll(context.TODO(), repo, repo, nil, + func(id string, sn *restic.Snapshot, err error) error { + if err == nil { + found = *sn + count++ + } + return nil + })) + test.Assert(t, count == 1, "unexpected number of snapshots: %v, wanted %v", count, 1) + test.Assert(t, desiredSnapshot.ID().Equal(*found.ID()), "unexpected snapshot found: %s, wanted %s", desiredSnapshot, found) +} From ff681b84f0ecd0fd5809222a1924cc18aaa793d0 Mon Sep 17 00:00:00 2001 From: Kjetil Torgrim Homme Date: Tue, 14 Jan 2025 01:40:35 +0100 Subject: [PATCH 2/7] lint: improve comment based on lint feedback --- internal/restic/duration.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/restic/duration.go b/internal/restic/duration.go index ed5acf24e..f10d1e811 100644 --- a/internal/restic/duration.go +++ b/internal/restic/duration.go @@ -127,7 +127,7 @@ func (d *Duration) Set(s string) error { return nil } -// Subtract duration from t to return past point in time. If t is not given, use time.Now(). +// ParseTime returns past point in time where this duration is relative to time t. If t is not given, use time.Now(). func (d Duration) PastTime(t ...time.Time) time.Time { var reftime time.Time if t == nil { From 96cdc02d668acff229f2d5ea23735b5ee07ac62f Mon Sep 17 00:00:00 2001 From: Kjetil Torgrim Homme Date: Tue, 14 Jan 2025 01:42:31 +0100 Subject: [PATCH 3/7] lint: ignore err from Time.parse which can not happen more idiomatically --- internal/restic/duration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/restic/duration_test.go b/internal/restic/duration_test.go index 60a618f08..fdafb173a 100644 --- a/internal/restic/duration_test.go +++ b/internal/restic/duration_test.go @@ -112,7 +112,7 @@ func TestPastTime(t *testing.T) { if err != nil { t.Fatal(err) } - reftime, err := time.Parse(time.DateTime, "1999-12-30 15:16:17") + reftime, _ := time.Parse(time.DateTime, "1999-12-30 15:16:17") expected := "1998-10-27 11:16:17" result := d.PastTime(reftime).Format(time.DateTime) if result != expected { From 11e4e5833b7ce8fe2414bc7d5156e2b2318a78ed Mon Sep 17 00:00:00 2001 From: Kjetil Torgrim Homme Date: Tue, 14 Jan 2025 02:04:30 +0100 Subject: [PATCH 4/7] add changelog for issue --- changelog/unreleased/issue-5136 | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 changelog/unreleased/issue-5136 diff --git a/changelog/unreleased/issue-5136 b/changelog/unreleased/issue-5136 new file mode 100644 index 000000000..40ed18853 --- /dev/null +++ b/changelog/unreleased/issue-5136 @@ -0,0 +1,10 @@ +Enhancement: Add snapshot filtering options --older-than and --newer-than + +Restic subcommands which work on a set of snapshots (``copy``, +``find``, ``forget``, ``snapshots``, ``tag`` and more) can now filter +on the absolute age of the snapshots specified as a duration, e.g., +``--newer-than 1d12h`` to only operate on snapshots created less than +36 hours ago, or ``--older-than 90d`` to operate on snapshots older +than three months. + +https://github.com/restic/restic/issues/5136 From 2585c68d168db819549f315e17ea336015c95cbc Mon Sep 17 00:00:00 2001 From: Kjetil Torgrim Homme Date: Tue, 14 Jan 2025 02:04:52 +0100 Subject: [PATCH 5/7] add hints about new functionality in documention --- doc/045_working_with_repos.rst | 3 +++ doc/060_forget.rst | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index d5f2240b8..2ca544d70 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -34,6 +34,9 @@ size of the contained files at the time when the snapshot was created. 590c8fc8 2015-05-08 21:47:38 kazik /srv 580.200MiB 9f0bc19e 2015-05-08 21:46:11 luigi /srv 572.180MiB +If you have many snapshots, you can restrict it to only snapshots made +the last week, using ``-newer-than 7d``. + You can filter the listing by directory path: .. code-block:: console diff --git a/doc/060_forget.rst b/doc/060_forget.rst index b211148cb..f0d45be34 100644 --- a/doc/060_forget.rst +++ b/doc/060_forget.rst @@ -353,7 +353,9 @@ Since restic 0.17.0, it is possible to delete all snapshots for a specific host, tag or path using the ``--unsafe-allow-remove-all`` option. The option must always be combined with a snapshot filter (by host, path or tag). For example the command ``forget --tag example --unsafe-allow-remove-all`` -removes all snapshots with tag ``example``. +removes all snapshots with tag ``example``. Since restic 0.17.4, this +can be combined with an option like ``--older-than 1y`` to implement +a hard maximum retention for snapshots. Security considerations in append-only mode From e588ee5bbef5f2e9f7e0d15ebdf7c499c20a33e2 Mon Sep 17 00:00:00 2001 From: Kjetil Torgrim Homme Date: Tue, 14 Jan 2025 02:09:32 +0100 Subject: [PATCH 6/7] lint: typo in previous comment fix --- internal/restic/duration.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/restic/duration.go b/internal/restic/duration.go index f10d1e811..292ba80fd 100644 --- a/internal/restic/duration.go +++ b/internal/restic/duration.go @@ -127,7 +127,7 @@ func (d *Duration) Set(s string) error { return nil } -// ParseTime returns past point in time where this duration is relative to time t. If t is not given, use time.Now(). +// PastTime returns past point in time where this duration is relative to time t. If t is not given, use time.Now(). func (d Duration) PastTime(t ...time.Time) time.Time { var reftime time.Time if t == nil { From 238d65ec3ffe64f63e718e483d1577aac5966248 Mon Sep 17 00:00:00 2001 From: Kjetil Torgrim Homme Date: Mon, 10 Feb 2025 20:12:37 +0100 Subject: [PATCH 7/7] SnapshotFilter is not empty if either NewerThan or OlderThan is set This is required to allow `forget` without other snapshot limitations --- internal/restic/snapshot_find.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/restic/snapshot_find.go b/internal/restic/snapshot_find.go index 39d8765d4..ee00cc9bc 100644 --- a/internal/restic/snapshot_find.go +++ b/internal/restic/snapshot_find.go @@ -29,7 +29,7 @@ type SnapshotFilter struct { } func (f *SnapshotFilter) Empty() bool { - return len(f.Hosts)+len(f.Tags)+len(f.Paths) == 0 + return len(f.Hosts)+len(f.Tags)+len(f.Paths) == 0 && f.NewerThan.Zero() && f.OlderThan.Zero() } func (f *SnapshotFilter) matches(sn *Snapshot) bool {