mirror of
https://github.com/restic/restic.git
synced 2025-03-09 00:00:02 +01:00
restic check with snapshot list
reworked the code using snapshotFilter.FindAll to find all snapshots and restic.FindUsedBlobs to retrieve all used blobs. range repo.LookupBlob (as before) to convert the blobs to their containing packfiles and c.repo.List(ctx, restic.PackFile, ...) to retrieve the sizes of those packfiles. Additional documentation and tests are still outstanding.
This commit is contained in:
parent
46184bd703
commit
c141ed1a17
2 changed files with 45 additions and 68 deletions
|
@ -390,26 +390,10 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
|
|||
|
||||
filterBySnapshot := false
|
||||
if len(args) > 0 || !opts.SnapshotFilter.Empty() {
|
||||
snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
|
||||
filterBySnapshot, err = chkr.CheckWithSnapshots(ctx, repo, args, &opts.SnapshotFilter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
visitedTrees := restic.NewIDSet()
|
||||
for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) {
|
||||
err := chkr.FindDataPackfiles(ctx, repo, sn, visitedTrees)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
filterBySnapshot = true
|
||||
}
|
||||
|
||||
selectedPacksSize := int64(0)
|
||||
for _, size := range chkr.GetPacks() {
|
||||
selectedPacksSize += size
|
||||
}
|
||||
printer.P("snapshot checking: %d packfiles with size %s selected.\n",
|
||||
chkr.CountPacks(), ui.FormatBytes(uint64(selectedPacksSize)))
|
||||
}
|
||||
|
||||
doReadData := func(packs map[restic.ID]int64) {
|
||||
|
|
|
@ -29,7 +29,6 @@ type Checker struct {
|
|||
sync.Mutex
|
||||
M restic.BlobSet
|
||||
}
|
||||
packSet restic.IDSet
|
||||
trackUnused bool
|
||||
|
||||
masterIndex *index.MasterIndex
|
||||
|
@ -42,7 +41,6 @@ type Checker struct {
|
|||
func New(repo restic.Repository, trackUnused bool) *Checker {
|
||||
c := &Checker{
|
||||
packs: make(map[restic.ID]int64),
|
||||
packSet: restic.NewIDSet(),
|
||||
masterIndex: index.NewMasterIndex(),
|
||||
repo: repo,
|
||||
trackUnused: trackUnused,
|
||||
|
@ -433,24 +431,12 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, er
|
|||
|
||||
// CountPacks returns the number of packs in the repository.
|
||||
func (c *Checker) CountPacks() uint64 {
|
||||
if len(c.packSet) == 0 {
|
||||
return uint64(len(c.packs))
|
||||
} else {
|
||||
return uint64(len(c.packSet))
|
||||
}
|
||||
return uint64(len(c.packs))
|
||||
}
|
||||
|
||||
// GetPacks returns IDSet of packs in the repository
|
||||
func (c *Checker) GetPacks() map[restic.ID]int64 {
|
||||
if len(c.packSet) == 0 {
|
||||
return c.packs
|
||||
} else {
|
||||
result := map[restic.ID]int64{}
|
||||
for packID := range c.packSet {
|
||||
result[packID] = c.packs[packID]
|
||||
}
|
||||
return result
|
||||
}
|
||||
return c.packs
|
||||
}
|
||||
|
||||
// ReadData loads all data from the repository and checks the integrity.
|
||||
|
@ -515,7 +501,6 @@ func (c *Checker) ReadPacks(ctx context.Context, packs map[restic.ID]int64, p *p
|
|||
for pack := range packs {
|
||||
packSet.Insert(pack)
|
||||
}
|
||||
|
||||
// push packs to ch
|
||||
for pbs := range c.repo.ListPacksFromIndex(ctx, packSet) {
|
||||
size := packs[pbs.PackID]
|
||||
|
@ -537,47 +522,55 @@ func (c *Checker) ReadPacks(ctx context.Context, packs map[restic.ID]int64, p *p
|
|||
}
|
||||
}
|
||||
|
||||
// Find data packfiles for repository checking based on snapshots.
|
||||
// Use restic.StreamTrees to gather all data blobs and convert them to their
|
||||
// containing packfile
|
||||
func (c *Checker) FindDataPackfiles(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot,
|
||||
visitedTrees restic.IDSet) error {
|
||||
// process snapshot IDs from command line
|
||||
func (c *Checker) CheckWithSnapshots(ctx context.Context, repo *repository.Repository, args []string, snapshotFilter *restic.SnapshotFilter) (bool, error) {
|
||||
|
||||
var packfileMutex sync.Mutex
|
||||
wg, wgCtx := errgroup.WithContext(ctx)
|
||||
treeStream := restic.StreamTrees(wgCtx, wg, repo, restic.IDs{*sn.Tree}, func(tree restic.ID) bool {
|
||||
visited := visitedTrees.Has(tree)
|
||||
visitedTrees.Insert(tree)
|
||||
return visited
|
||||
}, nil)
|
||||
|
||||
wg.Go(func() error {
|
||||
for tree := range treeStream {
|
||||
if tree.Error != nil {
|
||||
return fmt.Errorf("LoadTree(%v) returned error %v", tree.ID.Str(), tree.Error)
|
||||
}
|
||||
|
||||
packfileMutex.Lock()
|
||||
for _, node := range tree.Nodes {
|
||||
// Recursion into directories is handled by StreamTrees
|
||||
for _, content := range node.Content {
|
||||
result := repo.LookupBlob(restic.DataBlob, content)
|
||||
if len(result) == 0 {
|
||||
return fmt.Errorf("checker.LookupBlob: datablob %s not mapped!", content.Str())
|
||||
}
|
||||
c.packSet.Insert(result[0].PackID)
|
||||
}
|
||||
}
|
||||
packfileMutex.Unlock()
|
||||
selectedTrees := []restic.ID{}
|
||||
err := snapshotFilter.FindAll(ctx, c.snapshots, repo, args, func(id string, sn *restic.Snapshot, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
} else if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
selectedTrees = append(selectedTrees, *sn.Tree)
|
||||
return nil
|
||||
})
|
||||
|
||||
err := wg.Wait()
|
||||
if err != nil {
|
||||
return err
|
||||
return false, err
|
||||
}
|
||||
|
||||
return nil
|
||||
// gather used blobs from all trees
|
||||
usedBlobs := restic.NewBlobSet()
|
||||
err = restic.FindUsedBlobs(ctx, repo, selectedTrees, usedBlobs, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if len(selectedTrees) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// convert blobs to packfile IDs
|
||||
c.packs = map[restic.ID]int64{}
|
||||
for blob := range usedBlobs {
|
||||
for _, res := range repo.LookupBlob(blob.Type, blob.ID) {
|
||||
c.packs[res.PackID] = 0
|
||||
}
|
||||
}
|
||||
|
||||
// gather size for selected packfiles
|
||||
err = c.repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
|
||||
if _, ok := c.packs[id]; ok {
|
||||
c.packs[id] = size
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return len(selectedTrees) > 0, nil
|
||||
}
|
||||
|
|
Loading…
Add table
Reference in a new issue