From 4ffb6f2c5597c74beba54688d9fe352bb6f50fbd Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Sun, 2 Mar 2025 08:41:12 +0000 Subject: [PATCH 01/23] new implementation of limiting repository size during backup Changed saveAndEncrypt(...) to include tracking of current size and raising an error "MaxCapacityExceeded" when the limit has been exceeded. Added CurrentRepositorySize(ctx) to report current size Added MaxCapacityExceeded() to query if limit has been execced Added IsRepositoryLimitActive() to query if size monitoring is active In addition an interface definition added to fulfill the needs of internal/archiver for accessing repository functions. The issue descrition has been updated. --- changelog/unreleased/issue-4585 | 29 +++++++++++++++ internal/repository/repository.go | 59 ++++++++++++++++++++++++++++++- 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/issue-4585 diff --git a/changelog/unreleased/issue-4585 b/changelog/unreleased/issue-4585 new file mode 100644 index 000000000..bcea9c83d --- /dev/null +++ b/changelog/unreleased/issue-4585 @@ -0,0 +1,29 @@ +Enhancement: Limit repository size to a predefined maximum size during vackup + +Restic backup can now limit the repository size to a given maximum value. The +repository size limit can be specified as option `--max-repo-size`, with +the usual meaning for size related options. + +During backup, the current size is monitored by calculating its actual repository +size plus the size of any new blob added during the backup process. Once the +defined limit is exceeded, backup is prevented from creating new backup entries +and the 'in progress' files are finalized, without adding large amounts of new data. + +The size limit is a rough limit and cannot be taken as a precise limit, +since indexes and snapshot file have to be finalized. +Due to the highly parallel processing of a backup process +the limit can be overshot by `packfile size` multiple times. +With a current default of 16m for a packfile, +you might experience differences of multiple megabytes between the expected limit +and the actual size when the backup process terminates. + +The new packfiles are created by the current backup process and a new snapshot +will be created. + +Currently the implemenation is incomplete. There are currently two indications +that there is a incomplete backup: +field `PartialSnapshot` exist and is set to true and +the snapshot is tagged as `partial-snapshot`. + +https://github.com/restic/restic/issues/4583 +https://github.com/restic/restic/pull/5215 diff --git a/internal/repository/repository.go b/internal/repository/repository.go index aee0db103..c1243d5ef 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -51,6 +51,8 @@ type Repository struct { allocDec sync.Once enc *zstd.Encoder dec *zstd.Decoder + + MaxRepoCapReached bool } // internalRepository allows using SaveUnpacked and RemoveUnpacked with all FileTypes @@ -62,6 +64,8 @@ type Options struct { Compression CompressionMode PackSize uint NoExtraVerify bool + RepoSizeMax uint64 + repoCurSize uint64 } // CompressionMode configures if data should be compressed. @@ -391,7 +395,60 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data panic(fmt.Sprintf("invalid type: %v", t)) } - return pm.SaveBlob(ctx, t, id, ciphertext, uncompressedLength) + length, err := pm.SaveBlob(ctx, t, id, ciphertext, uncompressedLength) + + var m sync.Mutex + + // maximum repository capacity exceeded? + m.Lock() + defer m.Unlock() + if r.opts.RepoSizeMax > 0 { + r.opts.repoCurSize += uint64(length) + if r.opts.repoCurSize > r.opts.RepoSizeMax { + r.MaxRepoCapReached = true + debug.Log("MaxCapacityExceeded") + return length, errors.New("MaxCapacityExceeded") + } + } + return length, err +} + +// CurrentRepositorySize counts the sizes of the filetypes snapshot, index and packs +func (r *Repository) CurrentRepositorySize(ctx context.Context) (uint64, error) { + curSize := uint64(0) + if r.opts.RepoSizeMax > 0 { + for _, ft := range []restic.FileType{restic.SnapshotFile, restic.IndexFile, restic.PackFile} { + err := r.List(ctx, ft, func(_ restic.ID, size int64) error { + curSize += uint64(size) + return nil + }) + if err != nil { + return 0, err + } + } + r.opts.repoCurSize = curSize + return curSize, nil + } + + return 0, errors.New("repository maximum size has not been set") +} + +// MaxCapacityExceeded reports if repository has a limit and if it is exceeded +func (r *Repository) MaxCapacityExceeded() bool { + if r.opts.RepoSizeMax == 0 { + return false + } + return r.MaxRepoCapReached +} + +func (r *Repository) IsRepositoryLimitActive() bool { + return r.opts.RepoSizeMax > 0 +} + +// CapacityChecker has to satisfy restic.Repository interface needs +type CapacityChecker interface { + MaxCapacityExceeded() bool + IsRepositoryLimitActive() bool } func (r *Repository) verifyCiphertext(buf []byte, uncompressedLength int, id restic.ID) error { From 3d024514986dd6e7a8fa0bc6a3911d1761f9e804 Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Sun, 2 Mar 2025 08:57:52 +0000 Subject: [PATCH 02/23] In preparation for `restic backup --max-repo-size` extend interface definition of repository to include: MaxCapacityExceeded() bool IsRepositoryLimitActive() bool --- internal/restic/repository.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 977950f59..e5f9bed02 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -63,6 +63,11 @@ type Repository interface { // StartWarmup creates a new warmup job, requesting the backend to warmup the specified packs. StartWarmup(ctx context.Context, packs IDSet) (WarmupJob, error) + + // MaxCapacityExceeded checks if repository capacity has been exceeded + MaxCapacityExceeded() bool + // IsRepositoryLimitActive checks if maximum repository size monitoring is active + IsRepositoryLimitActive() bool } type FileType = backend.FileType From bf227e6237b805abadc92ae868577b43743fd8a8 Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Sun, 2 Mar 2025 09:05:49 +0000 Subject: [PATCH 03/23] Add field RepoSizeMax to global.go, Add field PartialSnapshot to Snapshot in preparation to limit repository size during backup. --- cmd/restic/global.go | 2 ++ internal/restic/snapshot.go | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index a8270e20d..f5b646dd5 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -73,6 +73,7 @@ type GlobalOptions struct { PackSize uint NoExtraVerify bool InsecureNoPassword bool + RepoSizeMax uint64 backend.TransportOptions limiter.Limits @@ -485,6 +486,7 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi Compression: opts.Compression, PackSize: opts.PackSize * 1024 * 1024, NoExtraVerify: opts.NoExtraVerify, + RepoSizeMax: opts.RepoSizeMax, }) if err != nil { return nil, errors.Fatal(err.Error()) diff --git a/internal/restic/snapshot.go b/internal/restic/snapshot.go index f9cdf4daf..e84f1e45c 100644 --- a/internal/restic/snapshot.go +++ b/internal/restic/snapshot.go @@ -25,8 +25,9 @@ type Snapshot struct { Tags []string `json:"tags,omitempty"` Original *ID `json:"original,omitempty"` - ProgramVersion string `json:"program_version,omitempty"` - Summary *SnapshotSummary `json:"summary,omitempty"` + ProgramVersion string `json:"program_version,omitempty"` + PartialSnapshot bool `json:"partial_snapshot,omitempty"` + Summary *SnapshotSummary `json:"summary,omitempty"` id *ID // plaintext ID, used during restore } From 612e21d42b2d1310e608caf3ace0872b564cb6f1 Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Sun, 2 Mar 2025 09:09:43 +0000 Subject: [PATCH 04/23] restic backup - handle MaxCapacityExceeded while saving a blob If the capacity size limit condition is activated, a TreeBlob is passed through unharmed, so the new tree can be saved properly. A DataBlob however, which could be very large in size will be replaced by the data blob which contains the string "MaxCapacityExceeded\n". --- internal/archiver/blob_saver.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/internal/archiver/blob_saver.go b/internal/archiver/blob_saver.go index 356a32ce2..96da5b008 100644 --- a/internal/archiver/blob_saver.go +++ b/internal/archiver/blob_saver.go @@ -68,6 +68,16 @@ type saveBlobResponse struct { func (s *blobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte) (saveBlobResponse, error) { id, known, sizeInRepo, err := s.repo.SaveBlob(ctx, t, buf, restic.ID{}, false) + if err != nil && t == restic.TreeBlob && err.Error() == "MaxCapacityExceeded" { + err = nil + } + + // need to modify data for repository monitoring being triggered + if err != nil && t == restic.DataBlob && err.Error() == "MaxCapacityExceeded" { + buf = []byte("MaxCapacityExceeded\n") + id = restic.Hash(buf) + return saveBlobResponse{id: id}, err + } if err != nil { return saveBlobResponse{}, err @@ -95,6 +105,10 @@ func (s *blobSaver) worker(ctx context.Context, jobs <-chan saveBlobJob) error { } res, err := s.saveBlob(ctx, job.BlobType, job.buf.Data) + // pass through unharmed for repository monitoring + if err != nil && err.Error() == "MaxCapacityExceeded" { + err = nil + } if err != nil { debug.Log("saveBlob returned error, exiting: %v", err) return fmt.Errorf("failed to save blob from file %q: %w", job.fn, err) From 767d66474b878b71c11dc183f81e9d164eca2f0f Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Sun, 2 Mar 2025 13:01:14 +0000 Subject: [PATCH 05/23] changed logic for processing failing MaxCapacityExceeded data blobs intercept MaxCapacityExceeded errors for all blobs and convert them to "no error" for all blobs but DataBlobs. For DataBlobs insert an extra data blob once, with the signature "MaxCapacityExceeded\n" For all data blob instances with the above error, return a `saveBlobResponse` with an ID of restic.Hash("MaxCapacityExceeded\n") This done to maintain the integritry of all newly saved blobs. --- internal/archiver/blob_saver.go | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/internal/archiver/blob_saver.go b/internal/archiver/blob_saver.go index 96da5b008..7a58334d6 100644 --- a/internal/archiver/blob_saver.go +++ b/internal/archiver/blob_saver.go @@ -3,6 +3,7 @@ package archiver import ( "context" "fmt" + "sync" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" @@ -68,18 +69,26 @@ type saveBlobResponse struct { func (s *blobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte) (saveBlobResponse, error) { id, known, sizeInRepo, err := s.repo.SaveBlob(ctx, t, buf, restic.ID{}, false) - if err != nil && t == restic.TreeBlob && err.Error() == "MaxCapacityExceeded" { - err = nil - } + if err != nil && err.Error() == "MaxCapacityExceeded" { + if t != restic.DataBlob { + err = nil + } else { + err = nil + var once sync.Once + // need to modify data blob for repository monitoring being triggered + buf = []byte("MaxCapacityExceeded\n") + id = restic.Hash(buf) + once.Do(func() { + _, _, _, err = s.repo.SaveBlob(ctx, restic.DataBlob, buf, id, false) - // need to modify data for repository monitoring being triggered - if err != nil && t == restic.DataBlob && err.Error() == "MaxCapacityExceeded" { - buf = []byte("MaxCapacityExceeded\n") - id = restic.Hash(buf) - return saveBlobResponse{id: id}, err - } - - if err != nil { + }) + if err != nil && err.Error() != "MaxCapacityExceeded" { + debug.Log("failing at saving extra data blob: %v", err) + return saveBlobResponse{}, err + } + return saveBlobResponse{id: id}, err + } + } else if err != nil { return saveBlobResponse{}, err } From 62a8f1bd89b7ed3fa7b73f86d08982d1e5352bbd Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Sun, 2 Mar 2025 13:08:44 +0000 Subject: [PATCH 06/23] changes to internal/archiver/archiver: allow `restic backup` to work with a repo size limitation in saveDir() don't save the directory if the current backup is in repository shutdown mode In saveTree() don't descend into subdirectories if the current backup is in repository shutdown mode in Snapshot() add the flag PartialSnapshot to the Snapshot structure, also add tag "partial-snapshot" to the tag list. --- internal/archiver/archiver.go | 58 ++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index 0b71cbacf..4cff485bf 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -15,6 +15,7 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/feature" "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "golang.org/x/sync/errgroup" ) @@ -75,6 +76,7 @@ type archiverRepo interface { restic.Loader restic.BlobSaver restic.SaverUnpacked[restic.WriteableFileType] + repository.CapacityChecker Config() restic.Config StartPackUploader(ctx context.Context, wg *errgroup.Group) @@ -327,34 +329,37 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me // test if context has been cancelled if ctx.Err() != nil { debug.Log("context has been cancelled, aborting") + return futureNode{}, ctx.Err() } pathname := arch.FS.Join(dir, name) oldNode := previous.Find(name) snItem := join(snPath, name) - fn, excluded, err := arch.save(ctx, snItem, pathname, oldNode) + // don't save if we are in repository shutdown mode + if !arch.Repo.MaxCapacityExceeded() { + fn, excluded, err := arch.save(ctx, snItem, pathname, oldNode) - // return error early if possible - if err != nil { - err = arch.error(pathname, err) - if err == nil { - // ignore error + // return error early if possible + if err != nil { + err = arch.error(pathname, err) + if err == nil { + // ignore error + continue + } + + return futureNode{}, err + } + + if excluded { continue } - return futureNode{}, err + nodes = append(nodes, fn) } - - if excluded { - continue - } - - nodes = append(nodes, fn) } fn := arch.treeSaver.Save(ctx, snPath, dir, treeNode, nodes, complete) - return fn, nil } @@ -717,14 +722,17 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, return futureNode{}, 0, err } - // not a leaf node, archive subtree - fn, _, err := arch.saveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *restic.Node, is ItemStats) { - arch.trackItem(snItem, oldNode, n, is, time.Since(start)) - }) - if err != nil { - return futureNode{}, 0, err + // don't descend into subdirectories if we are in shutdown mode + if !arch.Repo.MaxCapacityExceeded() { + // not a leaf node, archive subtree + fn, _, err := arch.saveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *restic.Node, is ItemStats) { + arch.trackItem(snItem, oldNode, n, is, time.Since(start)) + }) + if err != nil { + return futureNode{}, 0, err + } + nodes = append(nodes, fn) } - nodes = append(nodes, fn) } fn := arch.treeSaver.Save(ctx, snPath, atree.FileInfoPath, node, nodes, complete) @@ -909,7 +917,6 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps debug.Log("error while saving tree: %v", err) return err } - return arch.Repo.Flush(ctx) }) err = wgUp.Wait() @@ -924,6 +931,9 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps } } + if arch.Repo.MaxCapacityExceeded() { + opts.Tags = append(opts.Tags, "partial-snapshot") + } sn, err := restic.NewSnapshot(targets, opts.Tags, opts.Hostname, opts.Time) if err != nil { return nil, restic.ID{}, nil, err @@ -934,6 +944,10 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps if opts.ParentSnapshot != nil { sn.Parent = opts.ParentSnapshot.ID() } + if arch.Repo.MaxCapacityExceeded() { + sn.PartialSnapshot = true + } + sn.Tree = &rootTreeID arch.summary.BackupEnd = time.Now() sn.Summary = &restic.SnapshotSummary{ From 8c35a6d76db3b2dee274204babbb3e39d3764f30 Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Sun, 2 Mar 2025 16:19:07 +0000 Subject: [PATCH 07/23] restic backup --max-repo-size Implementation of limiting the repository size during the backup process. Introduce option --max-repo-size, which is converted to field gopts.RepoSizeMax, and handed over to repository.New() in cmd/restic/global.go. The command prints the current size in Verbosef mode at the beginning and also at the end, when the repository size has been exceeded. --- cmd/restic/cmd_backup.go | 45 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 70c0d2fb9..5853bf903 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -41,6 +41,19 @@ func newBackupCommand() *cobra.Command { The "backup" command creates a new snapshot and saves the files and directories given as the arguments. +When the option --max-repo-size is used, and the current backup is about to exceed the +defined limit, the snapshot is consistent in itself but unusable. It is not garanteed that +any data for this snapshot is valid or usable for a restore or a dump. An indication +for such a backup is the tag 'partial-snapshot'. In addition, a new field +'parhial_snapshot' is set to true, when you list the snapshot with +'restic -r ... cat snapshot ' command. + +The actual size of the snapshot can be multiple megabytes of data beyond the +specified limit, due to the high parallelism of the 'restic backup' command. + +If however the current backup stays within its defined limits of --max-repo-size, +everything id fine and the backup can be used. + EXIT STATUS =========== @@ -103,6 +116,8 @@ type BackupOptions struct { ReadConcurrency uint NoScan bool SkipIfUnchanged bool + RepoMaxSizeString string + RepoSizeMax uint64 } func (opts *BackupOptions) AddFlags(f *pflag.FlagSet) { @@ -143,6 +158,7 @@ func (opts *BackupOptions) AddFlags(f *pflag.FlagSet) { f.BoolVar(&opts.ExcludeCloudFiles, "exclude-cloud-files", false, "excludes online-only cloud files (such as OneDrive Files On-Demand)") } f.BoolVar(&opts.SkipIfUnchanged, "skip-if-unchanged", false, "skip snapshot creation if identical to parent snapshot") + f.StringVar(&opts.RepoMaxSizeString, "max-repo-size", "", "`limit` maximum size of repository - absolute value in bytes with suffixes m/M, g/G, t/T, default unlimited") // parse read concurrency from env, on error the default value will be used readConcurrency, _ := strconv.ParseUint(os.Getenv("RESTIC_READ_CONCURRENCY"), 10, 32) @@ -506,6 +522,14 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter } } + if len(opts.RepoMaxSizeString) > 0 { + size, err := ui.ParseBytes(opts.RepoMaxSizeString) + if err != nil { + return err + } + gopts.RepoSizeMax = uint64(size) + } + if gopts.verbosity >= 2 && !gopts.JSON { Verbosef("open repository\n") } @@ -516,6 +540,18 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter } defer unlock() + // get current size of repository + if gopts.RepoSizeMax > 0 { + curRepoSize, err := repo.CurrentRepositorySize(ctx) + if err != nil { + return err + } + Verbosef("Current repository size is %s\n", ui.FormatBytes(curRepoSize)) + if curRepoSize >= gopts.RepoSizeMax { + return errors.Fatal("repository maximum size already exceeded") + } + } + var progressPrinter backup.ProgressPrinter if gopts.JSON { progressPrinter = backup.NewJSONProgress(term, gopts.verbosity) @@ -691,6 +727,15 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter return ErrInvalidSourceData } + if repo.MaxCapacityExceeded() { + Verbosef("repository maximum size has been exceeded\n") + curRepoSize, err := repo.CurrentRepositorySize(ctx) + if err != nil { + return err + } + Verbosef("Current repository size is %s\n", ui.FormatBytes(curRepoSize)) + } + // Return error if any return werr } From 1abf1aafc82159bcd4828099000daea9dd9f995c Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Mon, 3 Mar 2025 06:56:03 +0000 Subject: [PATCH 08/23] restic backup - remove unneeded code and improve error information remove unneded field RepoSizeMax Improve code at the end of the run showing if repository capacity has been exceeded and give current value. --- cmd/restic/cmd_backup.go | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 5853bf903..c0058d7c7 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -116,8 +116,7 @@ type BackupOptions struct { ReadConcurrency uint NoScan bool SkipIfUnchanged bool - RepoMaxSizeString string - RepoSizeMax uint64 + RepoMaxSize string } func (opts *BackupOptions) AddFlags(f *pflag.FlagSet) { @@ -158,7 +157,7 @@ func (opts *BackupOptions) AddFlags(f *pflag.FlagSet) { f.BoolVar(&opts.ExcludeCloudFiles, "exclude-cloud-files", false, "excludes online-only cloud files (such as OneDrive Files On-Demand)") } f.BoolVar(&opts.SkipIfUnchanged, "skip-if-unchanged", false, "skip snapshot creation if identical to parent snapshot") - f.StringVar(&opts.RepoMaxSizeString, "max-repo-size", "", "`limit` maximum size of repository - absolute value in bytes with suffixes m/M, g/G, t/T, default unlimited") + f.StringVar(&opts.RepoMaxSize, "max-repo-size", "", "`limit` maximum size of repository - absolute value in bytes with suffixes m/M, g/G, t/T, default unlimited") // parse read concurrency from env, on error the default value will be used readConcurrency, _ := strconv.ParseUint(os.Getenv("RESTIC_READ_CONCURRENCY"), 10, 32) @@ -522,8 +521,8 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter } } - if len(opts.RepoMaxSizeString) > 0 { - size, err := ui.ParseBytes(opts.RepoMaxSizeString) + if len(opts.RepoMaxSize) > 0 { + size, err := ui.ParseBytes(opts.RepoMaxSize) if err != nil { return err } @@ -721,21 +720,23 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter return errors.Fatalf("unable to save snapshot: %v", err) } + if repo.MaxCapacityExceeded() { + Printf("\n=========================================\n") + Printf("repository maximum size has been exceeded\n") + curRepoSize, err := repo.CurrentRepositorySize(ctx) + if err != nil { + return err + } + Printf("Current repository size is %s\n", ui.FormatBytes(curRepoSize)) + Printf("=========================================\n\n") + } + // Report finished execution progressReporter.Finish(id, summary, opts.DryRun) if !success { return ErrInvalidSourceData } - if repo.MaxCapacityExceeded() { - Verbosef("repository maximum size has been exceeded\n") - curRepoSize, err := repo.CurrentRepositorySize(ctx) - if err != nil { - return err - } - Verbosef("Current repository size is %s\n", ui.FormatBytes(curRepoSize)) - } - // Return error if any return werr } From 09fb25413406e8052c984d267d28a9aa68c01cee Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Mon, 3 Mar 2025 07:00:10 +0000 Subject: [PATCH 09/23] internal/archiver/blob_saver removed ugly hack to save data blobs in case of the error "MaxCapacityExceeded", now just ignore it and continue as before. --- internal/archiver/blob_saver.go | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/internal/archiver/blob_saver.go b/internal/archiver/blob_saver.go index 7a58334d6..51d53c9b1 100644 --- a/internal/archiver/blob_saver.go +++ b/internal/archiver/blob_saver.go @@ -3,7 +3,7 @@ package archiver import ( "context" "fmt" - "sync" + //"sync" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" @@ -70,24 +70,7 @@ type saveBlobResponse struct { func (s *blobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte) (saveBlobResponse, error) { id, known, sizeInRepo, err := s.repo.SaveBlob(ctx, t, buf, restic.ID{}, false) if err != nil && err.Error() == "MaxCapacityExceeded" { - if t != restic.DataBlob { - err = nil - } else { - err = nil - var once sync.Once - // need to modify data blob for repository monitoring being triggered - buf = []byte("MaxCapacityExceeded\n") - id = restic.Hash(buf) - once.Do(func() { - _, _, _, err = s.repo.SaveBlob(ctx, restic.DataBlob, buf, id, false) - - }) - if err != nil && err.Error() != "MaxCapacityExceeded" { - debug.Log("failing at saving extra data blob: %v", err) - return saveBlobResponse{}, err - } - return saveBlobResponse{id: id}, err - } + err = nil } else if err != nil { return saveBlobResponse{}, err } @@ -114,7 +97,6 @@ func (s *blobSaver) worker(ctx context.Context, jobs <-chan saveBlobJob) error { } res, err := s.saveBlob(ctx, job.BlobType, job.buf.Data) - // pass through unharmed for repository monitoring if err != nil && err.Error() == "MaxCapacityExceeded" { err = nil } From 5ae0914b2646eb535d6754deff88add73006d1f0 Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Tue, 4 Mar 2025 08:35:53 +0000 Subject: [PATCH 10/23] repository functions: repo/repository.go: print limit reached message once, remove unneedef function IsRepositoryLimitActive() restic/repository.go: remove unnneded interface definition --- internal/repository/repository.go | 11 ++++------- internal/restic/repository.go | 2 -- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index c1243d5ef..270c64e72 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -405,8 +405,10 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data if r.opts.RepoSizeMax > 0 { r.opts.repoCurSize += uint64(length) if r.opts.repoCurSize > r.opts.RepoSizeMax { - r.MaxRepoCapReached = true - debug.Log("MaxCapacityExceeded") + if !r.MaxRepoCapReached { + debug.Log("MaxCapacityExceeded") + r.MaxRepoCapReached = true + } return length, errors.New("MaxCapacityExceeded") } } @@ -441,14 +443,9 @@ func (r *Repository) MaxCapacityExceeded() bool { return r.MaxRepoCapReached } -func (r *Repository) IsRepositoryLimitActive() bool { - return r.opts.RepoSizeMax > 0 -} - // CapacityChecker has to satisfy restic.Repository interface needs type CapacityChecker interface { MaxCapacityExceeded() bool - IsRepositoryLimitActive() bool } func (r *Repository) verifyCiphertext(buf []byte, uncompressedLength int, id restic.ID) error { diff --git a/internal/restic/repository.go b/internal/restic/repository.go index e5f9bed02..c41aa3105 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -66,8 +66,6 @@ type Repository interface { // MaxCapacityExceeded checks if repository capacity has been exceeded MaxCapacityExceeded() bool - // IsRepositoryLimitActive checks if maximum repository size monitoring is active - IsRepositoryLimitActive() bool } type FileType = backend.FileType From 812bf976b9eef77318e7ed53ee68320df3e2f1e6 Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Tue, 4 Mar 2025 08:39:53 +0000 Subject: [PATCH 11/23] filesaver in repository size monitoring mode If in monitoring mode, an active file gets truncated by breaking out of the reead loop The incomplete file list gets fixed by truncating it to the last non-null ID The file_saver_test.go needs updating with the repo parameter. --- internal/archiver/file_saver.go | 23 ++++++++++++++++++++--- internal/archiver/file_saver_test.go | 4 +++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/internal/archiver/file_saver.go b/internal/archiver/file_saver.go index ca8ec2fbb..681c7eedd 100644 --- a/internal/archiver/file_saver.go +++ b/internal/archiver/file_saver.go @@ -29,11 +29,12 @@ type fileSaver struct { CompleteBlob func(bytes uint64) NodeFromFileInfo func(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*restic.Node, error) + repo archiverRepo } // newFileSaver returns a new file saver. A worker pool with fileWorkers is // started, it is stopped when ctx is cancelled. -func newFileSaver(ctx context.Context, wg *errgroup.Group, save saveBlobFn, pol chunker.Pol, fileWorkers, blobWorkers uint) *fileSaver { +func newFileSaver(ctx context.Context, wg *errgroup.Group, save saveBlobFn, pol chunker.Pol, fileWorkers, blobWorkers uint, repo archiverRepo) *fileSaver { ch := make(chan saveFileJob) debug.Log("new file saver with %v file workers and %v blob workers", fileWorkers, blobWorkers) @@ -47,6 +48,7 @@ func newFileSaver(ctx context.Context, wg *errgroup.Group, save saveBlobFn, pol ch: ch, CompleteBlob: func(uint64) {}, + repo: repo, } for i := uint(0); i < fileWorkers; i++ { @@ -126,11 +128,23 @@ func (s *fileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat if isCompleted { panic("completed twice") } - for _, id := range fnr.node.Content { - if id.IsNull() { + + fixEnd := false + firstIndex := 0 + for i, id := range fnr.node.Content { + if s.repo.MaxCapacityExceeded() && id.IsNull() { + fixEnd = true + firstIndex = i + break + } else if id.IsNull() { panic("completed file with null ID") } } + + if fixEnd { + fnr.node.Content = fnr.node.Content[:firstIndex] + debug.Log("truncating file %q", fnr.snPath) + } isCompleted = true finish(fnr) } @@ -202,6 +216,9 @@ func (s *fileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat node.Content = append(node.Content, restic.ID{}) lock.Unlock() + if s.repo.MaxCapacityExceeded() { + break + } s.saveBlob(ctx, restic.DataBlob, buf, target, func(sbr saveBlobResponse) { lock.Lock() if !sbr.known { diff --git a/internal/archiver/file_saver_test.go b/internal/archiver/file_saver_test.go index ce862f6fe..2a2db1d18 100644 --- a/internal/archiver/file_saver_test.go +++ b/internal/archiver/file_saver_test.go @@ -10,6 +10,7 @@ import ( "github.com/restic/chunker" "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" "golang.org/x/sync/errgroup" @@ -48,7 +49,8 @@ func startFileSaver(ctx context.Context, t testing.TB, fsInst fs.FS) (*fileSaver t.Fatal(err) } - s := newFileSaver(ctx, wg, saveBlob, pol, workers, workers) + repo := repository.TestRepository(t) + s := newFileSaver(ctx, wg, saveBlob, pol, workers, workers, repo) s.NodeFromFileInfo = func(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*restic.Node, error) { return meta.ToNode(ignoreXattrListError) } From 92fe006ee11cb09862c1904b7a3089816ffccc76 Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Tue, 4 Mar 2025 08:43:37 +0000 Subject: [PATCH 12/23] archiver in repository size monitoring mode archiver.go: add repo parameter to newFileSaver(), remove unneeded check for monitoring check for saveTree blob_saver.go: remove unneded monitoring check for worker() --- internal/archiver/archiver.go | 24 ++++++++++++------------ internal/archiver/blob_saver.go | 4 ---- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index 4cff485bf..dfe1dca20 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -156,6 +156,9 @@ type Options struct { // SaveTreeConcurrency sets how many trees are marshalled and saved to the // repo concurrently. SaveTreeConcurrency uint + + // RepoSizeMax > 0 signals repository size monitoring + RepoSizeMax uint64 } // ApplyDefaults returns a copy of o with the default options set for all unset @@ -329,7 +332,6 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me // test if context has been cancelled if ctx.Err() != nil { debug.Log("context has been cancelled, aborting") - return futureNode{}, ctx.Err() } @@ -722,17 +724,14 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, return futureNode{}, 0, err } - // don't descend into subdirectories if we are in shutdown mode - if !arch.Repo.MaxCapacityExceeded() { - // not a leaf node, archive subtree - fn, _, err := arch.saveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *restic.Node, is ItemStats) { - arch.trackItem(snItem, oldNode, n, is, time.Since(start)) - }) - if err != nil { - return futureNode{}, 0, err - } - nodes = append(nodes, fn) + // not a leaf node, archive subtree + fn, _, err := arch.saveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *restic.Node, is ItemStats) { + arch.trackItem(snItem, oldNode, n, is, time.Since(start)) + }) + if err != nil { + return futureNode{}, 0, err } + nodes = append(nodes, fn) } fn := arch.treeSaver.Save(ctx, snPath, atree.FileInfoPath, node, nodes, complete) @@ -840,9 +839,10 @@ func (arch *Archiver) runWorkers(ctx context.Context, wg *errgroup.Group) { arch.fileSaver = newFileSaver(ctx, wg, arch.blobSaver.Save, arch.Repo.Config().ChunkerPolynomial, - arch.Options.ReadConcurrency, arch.Options.SaveBlobConcurrency) + arch.Options.ReadConcurrency, arch.Options.SaveBlobConcurrency, arch.Repo) arch.fileSaver.CompleteBlob = arch.CompleteBlob arch.fileSaver.NodeFromFileInfo = arch.nodeFromFileInfo + //arch.fileSaver.repo = arch.Repo arch.treeSaver = newTreeSaver(ctx, wg, arch.Options.SaveTreeConcurrency, arch.blobSaver.Save, arch.Error) } diff --git a/internal/archiver/blob_saver.go b/internal/archiver/blob_saver.go index 51d53c9b1..6864b16e9 100644 --- a/internal/archiver/blob_saver.go +++ b/internal/archiver/blob_saver.go @@ -3,7 +3,6 @@ package archiver import ( "context" "fmt" - //"sync" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" @@ -97,9 +96,6 @@ func (s *blobSaver) worker(ctx context.Context, jobs <-chan saveBlobJob) error { } res, err := s.saveBlob(ctx, job.BlobType, job.buf.Data) - if err != nil && err.Error() == "MaxCapacityExceeded" { - err = nil - } if err != nil { debug.Log("saveBlob returned error, exiting: %v", err) return fmt.Errorf("failed to save blob from file %q: %w", job.fn, err) From 55f3c2e08c51b4a1a976ac112973403a73166efa Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Tue, 4 Mar 2025 16:50:01 +0000 Subject: [PATCH 13/23] restic backup with repository size monitoring: enlarged help description added function checkPartialSnapshot() so commands working on snapshots can check ig they can use a given partial snapshot as input. --- cmd/restic/cmd_backup.go | 66 ++++++++++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 12 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index c0058d7c7..a142d0aa8 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -42,17 +42,21 @@ The "backup" command creates a new snapshot and saves the files and directories given as the arguments. When the option --max-repo-size is used, and the current backup is about to exceed the -defined limit, the snapshot is consistent in itself but unusable. It is not garanteed that -any data for this snapshot is valid or usable for a restore or a dump. An indication -for such a backup is the tag 'partial-snapshot'. In addition, a new field -'parhial_snapshot' is set to true, when you list the snapshot with -'restic -r ... cat snapshot ' command. +defined limit, the snapshot is consistent in itself but unusable. It is safe to assume that +any data for this snapshot are NOT valid nor usable for a restore or a dump coomand. +An indication for such a backup is the tag 'partial-snapshot'. In addition, a new field +'partial_snapshot' Snapshot is set to true, when you list the snapshot with the command +'restic -r ... cat snapshot '. -The actual size of the snapshot can be multiple megabytes of data beyond the +The actual size of the snapshot can be several megabytes of data beyond the specified limit, due to the high parallelism of the 'restic backup' command. +Files which have been started to be backed up, but haven't completed yet +will be truncated as soon as the condition (repository size limit reached) is raised. +Backup file data will NOT be correct. + If however the current backup stays within its defined limits of --max-repo-size, -everything id fine and the backup can be used. +everything is fine and the backup can be used. EXIT STATUS =========== @@ -116,7 +120,8 @@ type BackupOptions struct { ReadConcurrency uint NoScan bool SkipIfUnchanged bool - RepoMaxSize string + + RepoMaxSize string } func (opts *BackupOptions) AddFlags(f *pflag.FlagSet) { @@ -157,7 +162,7 @@ func (opts *BackupOptions) AddFlags(f *pflag.FlagSet) { f.BoolVar(&opts.ExcludeCloudFiles, "exclude-cloud-files", false, "excludes online-only cloud files (such as OneDrive Files On-Demand)") } f.BoolVar(&opts.SkipIfUnchanged, "skip-if-unchanged", false, "skip snapshot creation if identical to parent snapshot") - f.StringVar(&opts.RepoMaxSize, "max-repo-size", "", "`limit` maximum size of repository - absolute value in bytes with suffixes m/M, g/G, t/T, default unlimited") + f.StringVar(&opts.RepoMaxSize, "max-repo-size", "", "`limit` maximum size of repository - absolute value in bytes with suffixes m/M, g/G, t/T, default no limit") // parse read concurrency from env, on error the default value will be used readConcurrency, _ := strconv.ParseUint(os.Getenv("RESTIC_READ_CONCURRENCY"), 10, 32) @@ -545,7 +550,9 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter if err != nil { return err } - Verbosef("Current repository size is %s\n", ui.FormatBytes(curRepoSize)) + if !gopts.JSON { + Verbosef("Current repository size is %s\n", ui.FormatBytes(curRepoSize)) + } if curRepoSize >= gopts.RepoSizeMax { return errors.Fatal("repository maximum size already exceeded") } @@ -665,7 +672,8 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter wg.Go(func() error { return sc.Scan(cancelCtx, targets) }) } - arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: opts.ReadConcurrency}) + arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: opts.ReadConcurrency, + RepoSizeMax: gopts.RepoSizeMax}) arch.SelectByName = selectByNameFilter arch.Select = selectFilter arch.WithAtime = opts.WithAtime @@ -720,7 +728,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter return errors.Fatalf("unable to save snapshot: %v", err) } - if repo.MaxCapacityExceeded() { + if !gopts.JSON && repo.MaxCapacityExceeded() { Printf("\n=========================================\n") Printf("repository maximum size has been exceeded\n") curRepoSize, err := repo.CurrentRepositorySize(ctx) @@ -731,6 +739,10 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter Printf("=========================================\n\n") } + if werr == nil && repo.MaxCapacityExceeded() { + werr = errors.Fatal("backup incomplete, repositoy capacity exceeded") + } + // Report finished execution progressReporter.Finish(id, summary, opts.DryRun) if !success { @@ -740,3 +752,33 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter // Return error if any return werr } + +func checkPartialSnapshot(sn *restic.Snapshot, checkType string, subCommand string) error { + found := false + if sn.PartialSnapshot { + found = true + } else { + for _, tag := range sn.Tags { + if tag == "partial-snapshot" { + found = true + break + } + } + } + + if !found { + return nil + } + + switch checkType { + case "error", "fatal": + return errors.Fatalf("selected snapshot %s cannot be used with the command %q because it is a partial snapshot", + sn.ID().Str(), subCommand) + case "warn": + Warnf("be aware that command %s may create unexpected results because %s is a partial snapshot\n", + subCommand, sn.ID().Str()) + return nil + default: + return errors.New("type %s is invalid") + } +} From e36b07f78bb864741464340ebf7b364a96b1e93e Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Tue, 4 Mar 2025 16:55:05 +0000 Subject: [PATCH 14/23] restic backup - gofmt got me here is the better version by gofmt --- cmd/restic/cmd_backup.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index a142d0aa8..c55e50115 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -771,14 +771,14 @@ func checkPartialSnapshot(sn *restic.Snapshot, checkType string, subCommand stri } switch checkType { - case "error", "fatal": - return errors.Fatalf("selected snapshot %s cannot be used with the command %q because it is a partial snapshot", - sn.ID().Str(), subCommand) - case "warn": - Warnf("be aware that command %s may create unexpected results because %s is a partial snapshot\n", - subCommand, sn.ID().Str()) - return nil - default: - return errors.New("type %s is invalid") + case "error", "fatal": + return errors.Fatalf("selected snapshot %s cannot be used with the command %q because it is a partial snapshot", + sn.ID().Str(), subCommand) + case "warn": + Warnf("be aware that command %s may create unexpected results because %s is a partial snapshot\n", + subCommand, sn.ID().Str()) + return nil + default: + return errors.New("type %s is invalid") } } From 1873780d41a0b3f13a3f095b762218c8cfebd02c Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Tue, 4 Mar 2025 16:58:03 +0000 Subject: [PATCH 15/23] the commands which will definitely not operate on partial snapshots --- cmd/restic/cmd_copy.go | 4 ++++ cmd/restic/cmd_dump.go | 4 ++++ cmd/restic/cmd_restore.go | 4 ++++ cmd/restic/cmd_rewrite.go | 5 +++++ 4 files changed, 17 insertions(+) diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 2ad5a464c..2c1dc3182 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -129,6 +129,10 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] if sn.Original != nil { srcOriginal = *sn.Original } + err = checkPartialSnapshot(sn, "fatal", "copy") + if err != nil { + return err + } if originalSns, ok := dstSnapshotByOriginal[srcOriginal]; ok { isCopy := false diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index 978b64616..5bb879a39 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -156,6 +156,10 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] if err != nil { return errors.Fatalf("failed to find snapshot: %v", err) } + err = checkPartialSnapshot(sn, "fatal", "dump") + if err != nil { + return err + } bar := newIndexProgress(gopts.Quiet, gopts.JSON) err = repo.LoadIndex(ctx, bar) diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index a29b8a19e..667212249 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -145,6 +145,10 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, if err != nil { return errors.Fatalf("failed to find snapshot: %v", err) } + err = checkPartialSnapshot(sn, "error", "restore") + if err != nil { + return err + } bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term) err = repo.LoadIndex(ctx, bar) diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index 4e5b39932..cf8ff5009 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -321,6 +321,11 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a changedCount := 0 for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) { + err = checkPartialSnapshot(sn, "fatal", "rewrite") + if err != nil { + return err + } + Verbosef("\n%v\n", sn) changed, err := rewriteSnapshot(ctx, repo, sn, opts) if err != nil { From be6a1445037ceb324a385f5f6d896c9d75b3874b Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Wed, 5 Mar 2025 21:44:34 +0000 Subject: [PATCH 16/23] restic diff, stats, tag - more command which vannot handle partial snapshots inserted check for partial snapshots --- cmd/restic/cmd_diff.go | 4 ++++ cmd/restic/cmd_stats.go | 8 ++++++-- cmd/restic/cmd_tag.go | 4 ++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index 35fe97fbb..c7df552ac 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -74,6 +74,10 @@ func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.LoaderUnpac if err != nil { return nil, "", errors.Fatal(err.Error()) } + err = checkPartialSnapshot(sn, "fatal", "diff") + if err != nil { + return sn, "", err + } return sn, subfolder, err } diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 1e7df11cb..59366f6f2 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -29,13 +29,13 @@ func newStatsCommand() *cobra.Command { Short: "Scan the repository and show basic statistics", Long: ` The "stats" command walks one or multiple snapshots in a repository -and accumulates statistics about the data stored therein. It reports +and accumulates statistics about the data stored therein. It reports on the number of unique files and their sizes, according to one of the counting modes as given by the --mode flag. It operates on all snapshots matching the selection criteria or all snapshots if nothing is specified. The special snapshot ID "latest" -is also supported. Some modes make more sense over +is also supported. Some modes make more sense over just a single snapshot, while others are useful across all snapshots, depending on what you are trying to calculate. @@ -130,6 +130,10 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args } for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) { + err = checkPartialSnapshot(sn, "fatal", "stats") + if err != nil { + return err + } err = statsWalkSnapshot(ctx, sn, repo, opts, stats) if err != nil { return fmt.Errorf("error walking snapshot: %v", err) diff --git a/cmd/restic/cmd_tag.go b/cmd/restic/cmd_tag.go index 39e9a16b5..ca5b2144f 100644 --- a/cmd/restic/cmd_tag.go +++ b/cmd/restic/cmd_tag.go @@ -157,6 +157,10 @@ func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, term *ter } for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) { + err := checkPartialSnapshot(sn, "fatal", "tag") + if err != nil { + return err + } changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten(), printFunc) if err != nil { Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err) From b264f0660a548926423e09959d980f91b9d47673 Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Wed, 5 Mar 2025 21:55:28 +0000 Subject: [PATCH 17/23] restic prune - added function findPartialSnapshots() findPartialSnapshots collects all partial snapshots and sends the off to restic.ParallelRemove() to forget them before prune starts working. --- cmd/restic/cmd_prune.go | 51 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index a613f2255..54c488f7a 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -29,6 +29,9 @@ func newPruneCommand() *cobra.Command { The "prune" command checks the repository and removes data that is not referenced and therefore not needed any more. +The "prune" command automatically eliminates pertial snapshots since they take +up space and cannot really be used to do some usefull work. + EXIT STATUS =========== @@ -162,6 +165,12 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, term } defer unlock() + // check for partial snapshots - and remove them + err = findPartialSnapshots(ctx, repo, gopts, term) + if err != nil { + return err + } + if opts.UnsafeNoSpaceRecovery != "" { repoID := repo.Config().ID if opts.UnsafeNoSpaceRecovery != repoID { @@ -292,3 +301,45 @@ func getUsedBlobs(ctx context.Context, repo restic.Repository, usedBlobs restic. return restic.FindUsedBlobs(ctx, repo, snapshotTrees, usedBlobs, bar) } + +// findPartialSnapshots find all partial snapshots and 'forget' them +func findPartialSnapshots(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, term *termstatus.Terminal) error { + snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile) + if err != nil { + return err + } + + selectedSnaps := restic.IDSet{} + err = (&restic.SnapshotFilter{Tags: restic.TagLists{restic.TagList{"partial-snapshot"}}}).FindAll(ctx, snapshotLister, repo, []string{}, func(_ string, sn *restic.Snapshot, err error) error { + if err != nil { + return err + } + + selectedSnaps.Insert(*sn.ID()) + return nil + }) + if err != nil { + return err + } else if len(selectedSnaps) == 0 { + return nil + } + + // run forget + verbosity := gopts.verbosity + if gopts.JSON { + verbosity = 0 + } + printer := newTerminalProgressPrinter(verbosity, term) + bar := printer.NewCounter("partial snapshots deleted") + err = restic.ParallelRemove(ctx, repo, selectedSnaps, restic.WriteableSnapshotFile, func(id restic.ID, err error) error { + if err != nil { + printer.E("unable to remove partial snapshot %v/%v from the repository\n", restic.SnapshotFile, id) + } else { + printer.VV("removed partial snapshot %v/%v\n", restic.SnapshotFile, id) + } + return nil + }, bar) + bar.Done() + + return err +} From e7dddb065a3ea38fc30b8dc03dfad8aff4bf548b Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Wed, 5 Mar 2025 21:58:34 +0000 Subject: [PATCH 18/23] restic prune integration tests: added one test TestPruneSizeMonitoring() which runs a failed backup and then calls prune to remove the partial snapshot. cmd/restic/integration_test needed to be modified, since it does not allow multiple calls to list snapshots. --- cmd/restic/cmd_prune_integration_test.go | 30 ++++++++++++++++++++++-- cmd/restic/integration_test.go | 22 ++++++++++++++--- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/cmd/restic/cmd_prune_integration_test.go b/cmd/restic/cmd_prune_integration_test.go index 0561f8243..95b0c8e4a 100644 --- a/cmd/restic/cmd_prune_integration_test.go +++ b/cmd/restic/cmd_prune_integration_test.go @@ -8,13 +8,14 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/repository" + //"github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" "github.com/restic/restic/internal/ui/termstatus" ) func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) { oldHook := gopts.backendTestHook - gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil } + gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListMultipleBackend(r), nil } defer func() { gopts.backendTestHook = oldHook }() @@ -142,7 +143,7 @@ func TestPruneWithDamagedRepository(t *testing.T) { removePacksExcept(env.gopts, t, oldPacks, false) oldHook := env.gopts.backendTestHook - env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil } + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListMultipleBackend(r), nil } defer func() { env.gopts.backendTestHook = oldHook }() @@ -237,3 +238,28 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o "prune should have reported an error") } } + +func TestPruneSizeMonitoring(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + datafile := filepath.Join("testdata", "backup-data.tar.gz") + testRunInit(t, env.gopts) + + rtest.SetupTarTestFixture(t, env.testdata, datafile) + opts := BackupOptions{RepoMaxSize: "50k"} + + // create and delete snapshot to create unused blobs + oldHook := env.gopts.backendTestHook + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListMultipleBackend(r), nil } + defer func() { + env.gopts.backendTestHook = oldHook + }() + err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{env.testdata}, opts, env.gopts) + rtest.Assert(t, err != nil, "backup should have ended in failure '%v'", err) + firstSnapshot := testListSnapshots(t, env.gopts, 1)[0] + t.Logf("first snapshot %v", firstSnapshot) + + testRunPrune(t, env.gopts, PruneOptions{MaxUnused: "0"}) + _ = testListSnapshots(t, env.gopts, 0) +} diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 3ef98a168..2cbc0e9b8 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -46,15 +46,20 @@ type listOnceBackend struct { backend.Backend listedFileType map[restic.FileType]bool strictOrder bool + allowMultiple bool } -func newListOnceBackend(be backend.Backend) *listOnceBackend { +// the linter bites here: says newListOnceBackend is not used +// I need to be able to call SnapshotLister more than once to check for partial snapshots +// and restic prune uses `getUsedBlobs` to get all used blobs which uses +// `restic.ForAllSnapshots()` to do the work. +/*func newListOnceBackend(be backend.Backend) *listOnceBackend { return &listOnceBackend{ Backend: be, listedFileType: make(map[restic.FileType]bool), strictOrder: false, } -} +}*/ func newOrderedListOnceBackend(be backend.Backend) *listOnceBackend { return &listOnceBackend{ @@ -64,6 +69,15 @@ func newOrderedListOnceBackend(be backend.Backend) *listOnceBackend { } } +func newListMultipleBackend(be backend.Backend) *listOnceBackend { + return &listOnceBackend{ + Backend: be, + listedFileType: make(map[restic.FileType]bool), + strictOrder: false, + allowMultiple: true, + } +} + func (be *listOnceBackend) List(ctx context.Context, t restic.FileType, fn func(backend.FileInfo) error) error { if t != restic.LockFile && be.listedFileType[t] { return errors.Errorf("tried listing type %v the second time", t) @@ -71,7 +85,9 @@ func (be *listOnceBackend) List(ctx context.Context, t restic.FileType, fn func( if be.strictOrder && t == restic.SnapshotFile && be.listedFileType[restic.IndexFile] { return errors.Errorf("tried listing type snapshots after index") } - be.listedFileType[t] = true + if !be.allowMultiple { + be.listedFileType[t] = true + } return be.Backend.List(ctx, t, fn) } From 8e71a92976094538822a1c9b10583008c604ed80 Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Wed, 5 Mar 2025 22:09:58 +0000 Subject: [PATCH 19/23] repository/repository: fixed race condition i by having a global lock attached to the Repository structure --- internal/repository/repository.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 270c64e72..c7850c29d 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -52,7 +52,9 @@ type Repository struct { enc *zstd.Encoder dec *zstd.Decoder - MaxRepoCapReached bool + maxRepoCapReached bool + maxRepoMutex sync.Mutex + } // internalRepository allows using SaveUnpacked and RemoveUnpacked with all FileTypes @@ -400,14 +402,14 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data var m sync.Mutex // maximum repository capacity exceeded? - m.Lock() - defer m.Unlock() + r.maxRepoMutex.Lock() + defer r.maxRepoMutex.Unlock() if r.opts.RepoSizeMax > 0 { r.opts.repoCurSize += uint64(length) if r.opts.repoCurSize > r.opts.RepoSizeMax { - if !r.MaxRepoCapReached { + if !r.maxRepoCapReached { debug.Log("MaxCapacityExceeded") - r.MaxRepoCapReached = true + r.maxRepoCapReached = true } return length, errors.New("MaxCapacityExceeded") } @@ -440,7 +442,7 @@ func (r *Repository) MaxCapacityExceeded() bool { if r.opts.RepoSizeMax == 0 { return false } - return r.MaxRepoCapReached + return r.maxRepoCapReached } // CapacityChecker has to satisfy restic.Repository interface needs From d794e412529066b5deebcda0746ae4747d4eceb0 Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Wed, 5 Mar 2025 22:15:27 +0000 Subject: [PATCH 20/23] repository/repository - fix unused sync.Mutex m --- internal/repository/repository.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index c7850c29d..b2f566291 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -54,7 +54,6 @@ type Repository struct { maxRepoCapReached bool maxRepoMutex sync.Mutex - } // internalRepository allows using SaveUnpacked and RemoveUnpacked with all FileTypes @@ -399,8 +398,6 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data length, err := pm.SaveBlob(ctx, t, id, ciphertext, uncompressedLength) - var m sync.Mutex - // maximum repository capacity exceeded? r.maxRepoMutex.Lock() defer r.maxRepoMutex.Unlock() From 48510f920b8f5a396878a01bf2837c17cc185545 Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Wed, 5 Mar 2025 22:26:21 +0000 Subject: [PATCH 21/23] repository/repository - protect r.maxRepoCapReached by Lock() in MaxCapacityExceeded() --- internal/repository/repository.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index b2f566291..6d2ee19f9 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -436,6 +436,8 @@ func (r *Repository) CurrentRepositorySize(ctx context.Context) (uint64, error) // MaxCapacityExceeded reports if repository has a limit and if it is exceeded func (r *Repository) MaxCapacityExceeded() bool { + r.maxRepoMutex.Lock() + defer r.maxRepoMutex.Unlock() if r.opts.RepoSizeMax == 0 { return false } From 264878bbfad2eae28e60707755a3c931335f8579 Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Thu, 6 Mar 2025 16:37:20 +0000 Subject: [PATCH 22/23] restic backup - corrected typo in error message --- cmd/restic/cmd_backup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index c55e50115..6bb68f099 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -740,7 +740,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter } if werr == nil && repo.MaxCapacityExceeded() { - werr = errors.Fatal("backup incomplete, repositoy capacity exceeded") + werr = errors.Fatal("backup incomplete, repository capacity exceeded") } // Report finished execution From b3b30db0a2a2bf831f0245e303b85c76dea8de45 Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Thu, 6 Mar 2025 16:38:24 +0000 Subject: [PATCH 23/23] restic backup and restic prune: more / modified tests cmd_prune_integration_test: testing for the exact error message cmd_backup_integration_test: two additional backup test with repository size monitoring TestBackupRepoSizeMonitorOnFirstBackup(): limit first backup on size, check error message TestBackupRepoSizeMonitorOnSecondBackup(): do a normal first backup and then limit second backup, so size is already exceeded on start of backup: check for error message. --- cmd/restic/cmd_backup_integration_test.go | 54 +++++++++++++++++++++++ cmd/restic/cmd_prune_integration_test.go | 8 ++-- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/cmd/restic/cmd_backup_integration_test.go b/cmd/restic/cmd_backup_integration_test.go index 06d71e345..cb35bc0dd 100644 --- a/cmd/restic/cmd_backup_integration_test.go +++ b/cmd/restic/cmd_backup_integration_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "crypto/rand" "fmt" "io" "os" @@ -10,6 +11,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -723,3 +725,55 @@ func TestBackupSkipIfUnchanged(t *testing.T) { testRunCheck(t, env.gopts) } + +func TestBackupRepoSizeMonitorOnFirstBackup(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + datafile := filepath.Join("testdata", "backup-data.tar.gz") + testRunInit(t, env.gopts) + + rtest.SetupTarTestFixture(t, env.testdata, datafile) + opts := BackupOptions{RepoMaxSize: "50k"} + + // create and delete snapshot to create unused blobs + oldHook := env.gopts.backendTestHook + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListMultipleBackend(r), nil } + defer func() { + env.gopts.backendTestHook = oldHook + }() + err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{env.testdata}, opts, env.gopts) + rtest.Assert(t, err != nil && err.Error() == "Fatal: backup incomplete, repository capacity exceeded", + "failed as %q", err) +} + +func TestBackupRepoSizeMonitorOnSecondBackup(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + datafile := filepath.Join("testdata", "backup-data.tar.gz") + testRunInit(t, env.gopts) + + rtest.SetupTarTestFixture(t, env.testdata, datafile) + opts := BackupOptions{} + + // backup #1 + testRunBackup(t, filepath.Dir(env.testdata), []string{env.testdata}, opts, env.gopts) + testListSnapshots(t, env.gopts, 1) + + // insert new file into backup structure (8k) + createRandomDataFile(t, filepath.Join(env.testdata, "0", "0", "9", "rand.data")) + + // backup #2 + opts = BackupOptions{RepoMaxSize: "1k"} + err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{env.testdata}, opts, env.gopts) + rtest.Assert(t, err != nil && err.Error() == "Fatal: repository maximum size already exceeded", + "failed as %q", err) +} + +// make a random file, large enough to exceed backup size limit +func createRandomDataFile(t *testing.T, path string) { + randomData := make([]byte, 8*1024) + _, _ = rand.Read(randomData) + rtest.OK(t, os.WriteFile(path, randomData, 0600)) +} diff --git a/cmd/restic/cmd_prune_integration_test.go b/cmd/restic/cmd_prune_integration_test.go index 95b0c8e4a..77320622e 100644 --- a/cmd/restic/cmd_prune_integration_test.go +++ b/cmd/restic/cmd_prune_integration_test.go @@ -256,10 +256,10 @@ func TestPruneSizeMonitoring(t *testing.T) { env.gopts.backendTestHook = oldHook }() err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{env.testdata}, opts, env.gopts) - rtest.Assert(t, err != nil, "backup should have ended in failure '%v'", err) - firstSnapshot := testListSnapshots(t, env.gopts, 1)[0] - t.Logf("first snapshot %v", firstSnapshot) + rtest.Assert(t, err != nil && err.Error() == "Fatal: backup incomplete, repository capacity exceeded", + "failed as %q", err) + testListSnapshots(t, env.gopts, 1) testRunPrune(t, env.gopts, PruneOptions{MaxUnused: "0"}) - _ = testListSnapshots(t, env.gopts, 0) + testListSnapshots(t, env.gopts, 0) }