From c141ed1a17e6dafb5e63cdfcde583cba6d18d5b4 Mon Sep 17 00:00:00 2001 From: Winfried Plappert Date: Tue, 4 Feb 2025 18:45:03 +0000 Subject: [PATCH] restic check with snapshot list reworked the code using snapshotFilter.FindAll to find all snapshots and restic.FindUsedBlobs to retrieve all used blobs. range repo.LookupBlob (as before) to convert the blobs to their containing packfiles and c.repo.List(ctx, restic.PackFile, ...) to retrieve the sizes of those packfiles. Additional documentation and tests are still outstanding. --- cmd/restic/cmd_check.go | 18 +------ internal/checker/checker.go | 95 +++++++++++++++++-------------------- 2 files changed, 45 insertions(+), 68 deletions(-) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index 4edac3f9c..b1705348f 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -390,26 +390,10 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args filterBySnapshot := false if len(args) > 0 || !opts.SnapshotFilter.Empty() { - snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile) + filterBySnapshot, err = chkr.CheckWithSnapshots(ctx, repo, args, &opts.SnapshotFilter) if err != nil { return err } - - visitedTrees := restic.NewIDSet() - for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) { - err := chkr.FindDataPackfiles(ctx, repo, sn, visitedTrees) - if err != nil { - return err - } - filterBySnapshot = true - } - - selectedPacksSize := int64(0) - for _, size := range chkr.GetPacks() { - selectedPacksSize += size - } - printer.P("snapshot checking: %d packfiles with size %s selected.\n", - chkr.CountPacks(), ui.FormatBytes(uint64(selectedPacksSize))) } doReadData := func(packs map[restic.ID]int64) { diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 013563790..2ac2b3615 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -29,7 +29,6 @@ type Checker struct { sync.Mutex M restic.BlobSet } - packSet restic.IDSet trackUnused bool masterIndex *index.MasterIndex @@ -42,7 +41,6 @@ type Checker struct { func New(repo restic.Repository, trackUnused bool) *Checker { c := &Checker{ packs: make(map[restic.ID]int64), - packSet: restic.NewIDSet(), masterIndex: index.NewMasterIndex(), repo: repo, trackUnused: trackUnused, @@ -433,24 +431,12 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, er // CountPacks returns the number of packs in the repository. func (c *Checker) CountPacks() uint64 { - if len(c.packSet) == 0 { - return uint64(len(c.packs)) - } else { - return uint64(len(c.packSet)) - } + return uint64(len(c.packs)) } // GetPacks returns IDSet of packs in the repository func (c *Checker) GetPacks() map[restic.ID]int64 { - if len(c.packSet) == 0 { - return c.packs - } else { - result := map[restic.ID]int64{} - for packID := range c.packSet { - result[packID] = c.packs[packID] - } - return result - } + return c.packs } // ReadData loads all data from the repository and checks the integrity. @@ -515,7 +501,6 @@ func (c *Checker) ReadPacks(ctx context.Context, packs map[restic.ID]int64, p *p for pack := range packs { packSet.Insert(pack) } - // push packs to ch for pbs := range c.repo.ListPacksFromIndex(ctx, packSet) { size := packs[pbs.PackID] @@ -537,47 +522,55 @@ func (c *Checker) ReadPacks(ctx context.Context, packs map[restic.ID]int64, p *p } } -// Find data packfiles for repository checking based on snapshots. -// Use restic.StreamTrees to gather all data blobs and convert them to their -// containing packfile -func (c *Checker) FindDataPackfiles(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, - visitedTrees restic.IDSet) error { +// process snapshot IDs from command line +func (c *Checker) CheckWithSnapshots(ctx context.Context, repo *repository.Repository, args []string, snapshotFilter *restic.SnapshotFilter) (bool, error) { - var packfileMutex sync.Mutex - wg, wgCtx := errgroup.WithContext(ctx) - treeStream := restic.StreamTrees(wgCtx, wg, repo, restic.IDs{*sn.Tree}, func(tree restic.ID) bool { - visited := visitedTrees.Has(tree) - visitedTrees.Insert(tree) - return visited - }, nil) - - wg.Go(func() error { - for tree := range treeStream { - if tree.Error != nil { - return fmt.Errorf("LoadTree(%v) returned error %v", tree.ID.Str(), tree.Error) - } - - packfileMutex.Lock() - for _, node := range tree.Nodes { - // Recursion into directories is handled by StreamTrees - for _, content := range node.Content { - result := repo.LookupBlob(restic.DataBlob, content) - if len(result) == 0 { - return fmt.Errorf("checker.LookupBlob: datablob %s not mapped!", content.Str()) - } - c.packSet.Insert(result[0].PackID) - } - } - packfileMutex.Unlock() + selectedTrees := []restic.ID{} + err := snapshotFilter.FindAll(ctx, c.snapshots, repo, args, func(id string, sn *restic.Snapshot, err error) error { + if err != nil { + return err + } else if ctx.Err() != nil { + return ctx.Err() } + selectedTrees = append(selectedTrees, *sn.Tree) return nil }) - err := wg.Wait() if err != nil { - return err + return false, err } - return nil + // gather used blobs from all trees + usedBlobs := restic.NewBlobSet() + err = restic.FindUsedBlobs(ctx, repo, selectedTrees, usedBlobs, nil) + if err != nil { + return false, err + } + + if len(selectedTrees) == 0 { + return false, nil + } + + // convert blobs to packfile IDs + c.packs = map[restic.ID]int64{} + for blob := range usedBlobs { + for _, res := range repo.LookupBlob(blob.Type, blob.ID) { + c.packs[res.PackID] = 0 + } + } + + // gather size for selected packfiles + err = c.repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error { + if _, ok := c.packs[id]; ok { + c.packs[id] = size + } + return nil + }) + + if err != nil { + return false, err + } + + return len(selectedTrees) > 0, nil }