1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2025-03-09 00:00:02 +01:00

restic check with snapshot list

reworked the code using snapshotFilter.FindAll to find all snapshots and
restic.FindUsedBlobs to retrieve all used blobs.
range repo.LookupBlob (as before) to convert the blobs to their containing packfiles
and c.repo.List(ctx, restic.PackFile, ...) to retrieve the sizes of those packfiles.

Additional documentation and tests are still outstanding.
This commit is contained in:
Winfried Plappert 2025-02-04 18:45:03 +00:00
parent 46184bd703
commit c141ed1a17
2 changed files with 45 additions and 68 deletions

View file

@ -390,26 +390,10 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
filterBySnapshot := false
if len(args) > 0 || !opts.SnapshotFilter.Empty() {
snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
filterBySnapshot, err = chkr.CheckWithSnapshots(ctx, repo, args, &opts.SnapshotFilter)
if err != nil {
return err
}
visitedTrees := restic.NewIDSet()
for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) {
err := chkr.FindDataPackfiles(ctx, repo, sn, visitedTrees)
if err != nil {
return err
}
filterBySnapshot = true
}
selectedPacksSize := int64(0)
for _, size := range chkr.GetPacks() {
selectedPacksSize += size
}
printer.P("snapshot checking: %d packfiles with size %s selected.\n",
chkr.CountPacks(), ui.FormatBytes(uint64(selectedPacksSize)))
}
doReadData := func(packs map[restic.ID]int64) {

View file

@ -29,7 +29,6 @@ type Checker struct {
sync.Mutex
M restic.BlobSet
}
packSet restic.IDSet
trackUnused bool
masterIndex *index.MasterIndex
@ -42,7 +41,6 @@ type Checker struct {
func New(repo restic.Repository, trackUnused bool) *Checker {
c := &Checker{
packs: make(map[restic.ID]int64),
packSet: restic.NewIDSet(),
masterIndex: index.NewMasterIndex(),
repo: repo,
trackUnused: trackUnused,
@ -433,24 +431,12 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, er
// CountPacks returns the number of packs in the repository.
func (c *Checker) CountPacks() uint64 {
if len(c.packSet) == 0 {
return uint64(len(c.packs))
} else {
return uint64(len(c.packSet))
}
return uint64(len(c.packs))
}
// GetPacks returns IDSet of packs in the repository
func (c *Checker) GetPacks() map[restic.ID]int64 {
if len(c.packSet) == 0 {
return c.packs
} else {
result := map[restic.ID]int64{}
for packID := range c.packSet {
result[packID] = c.packs[packID]
}
return result
}
return c.packs
}
// ReadData loads all data from the repository and checks the integrity.
@ -515,7 +501,6 @@ func (c *Checker) ReadPacks(ctx context.Context, packs map[restic.ID]int64, p *p
for pack := range packs {
packSet.Insert(pack)
}
// push packs to ch
for pbs := range c.repo.ListPacksFromIndex(ctx, packSet) {
size := packs[pbs.PackID]
@ -537,47 +522,55 @@ func (c *Checker) ReadPacks(ctx context.Context, packs map[restic.ID]int64, p *p
}
}
// Find data packfiles for repository checking based on snapshots.
// Use restic.StreamTrees to gather all data blobs and convert them to their
// containing packfile
func (c *Checker) FindDataPackfiles(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot,
visitedTrees restic.IDSet) error {
// process snapshot IDs from command line
func (c *Checker) CheckWithSnapshots(ctx context.Context, repo *repository.Repository, args []string, snapshotFilter *restic.SnapshotFilter) (bool, error) {
var packfileMutex sync.Mutex
wg, wgCtx := errgroup.WithContext(ctx)
treeStream := restic.StreamTrees(wgCtx, wg, repo, restic.IDs{*sn.Tree}, func(tree restic.ID) bool {
visited := visitedTrees.Has(tree)
visitedTrees.Insert(tree)
return visited
}, nil)
wg.Go(func() error {
for tree := range treeStream {
if tree.Error != nil {
return fmt.Errorf("LoadTree(%v) returned error %v", tree.ID.Str(), tree.Error)
}
packfileMutex.Lock()
for _, node := range tree.Nodes {
// Recursion into directories is handled by StreamTrees
for _, content := range node.Content {
result := repo.LookupBlob(restic.DataBlob, content)
if len(result) == 0 {
return fmt.Errorf("checker.LookupBlob: datablob %s not mapped!", content.Str())
}
c.packSet.Insert(result[0].PackID)
}
}
packfileMutex.Unlock()
selectedTrees := []restic.ID{}
err := snapshotFilter.FindAll(ctx, c.snapshots, repo, args, func(id string, sn *restic.Snapshot, err error) error {
if err != nil {
return err
} else if ctx.Err() != nil {
return ctx.Err()
}
selectedTrees = append(selectedTrees, *sn.Tree)
return nil
})
err := wg.Wait()
if err != nil {
return err
return false, err
}
return nil
// gather used blobs from all trees
usedBlobs := restic.NewBlobSet()
err = restic.FindUsedBlobs(ctx, repo, selectedTrees, usedBlobs, nil)
if err != nil {
return false, err
}
if len(selectedTrees) == 0 {
return false, nil
}
// convert blobs to packfile IDs
c.packs = map[restic.ID]int64{}
for blob := range usedBlobs {
for _, res := range repo.LookupBlob(blob.Type, blob.ID) {
c.packs[res.PackID] = 0
}
}
// gather size for selected packfiles
err = c.repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
if _, ok := c.packs[id]; ok {
c.packs[id] = size
}
return nil
})
if err != nil {
return false, err
}
return len(selectedTrees) > 0, nil
}