Mirrors/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2025-03-09 00:00:02 +01:00
Code Issues Releases Wiki Activity

ui: restore: add progress and summary reporting for cloned/copied files

Browse source 
We try to distinguish truly block-cloned (i.e. "reflinked") files from
files that we failed to clone and instead had to use `io.Copy()` on,
because this might be useful to the operator/administrator to gauge the
space savings or gain awareness of filesystem configuration problems.
This commit is contained in:
Ivan Shapovalov 2024-12-04 02:28:35 +04:00
parent 47eb3d2cfe
commit 4e446c600a
7 changed files with 82 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
internal/restorer/restorer.go
View file

@ -329,7 +329,7 @@ func (res *Restorer) restoreReflink(node *restic.Node, target, path, location st
}
}
res.opts.Progress.AddProgress(location, restoreui.ActionFileRestored, node.Size, node.Size)
res.opts.Progress.AddClonedFile(location, node.Size, cloned)
// reflinked files *do* have separate metadata
return res.restoreNodeMetadataTo(node, path, location)
}

18
internal/ui/restore/json.go
View file

@ -34,9 +34,13 @@ func (t *jsonPrinter) Update(p State, duration time.Duration) {
FilesRestored: p.FilesFinished,
FilesSkipped: p.FilesSkipped,
FilesDeleted: p.FilesDeleted,
FilesCloned: p.FilesCloned,
FilesCopied: p.FilesCopied,
TotalBytes: p.AllBytesTotal,
BytesRestored: p.AllBytesWritten,
BytesSkipped: p.AllBytesSkipped,
BytesCloned: p.AllBytesCloned,
BytesCopied: p.AllBytesCopied,
}
if p.AllBytesTotal > 0 {
@ -71,6 +75,8 @@ func (t *jsonPrinter) CompleteItem(messageType ItemAction, item string, size uin
action = "restored"
case ActionFileUpdated:
action = "updated"
case ActionFileCloned:
action = "cloned"
case ActionFileUnchanged:
action = "unchanged"
case ActionDeleted:
@ -96,9 +102,13 @@ func (t *jsonPrinter) Finish(p State, duration time.Duration) {
FilesRestored: p.FilesFinished,
FilesSkipped: p.FilesSkipped,
FilesDeleted: p.FilesDeleted,
FilesCloned: p.FilesCloned,
FilesCopied: p.FilesCopied,
TotalBytes: p.AllBytesTotal,
BytesRestored: p.AllBytesWritten,
BytesSkipped: p.AllBytesSkipped,
BytesCloned: p.AllBytesCloned,
BytesCopied: p.AllBytesCopied,
}
t.print(status)
}
@ -111,9 +121,13 @@ type statusUpdate struct {
FilesRestored uint64 `json:"files_restored,omitempty"`
FilesSkipped uint64 `json:"files_skipped,omitempty"`
FilesDeleted uint64 `json:"files_deleted,omitempty"`
FilesCloned uint64 `json:"files_cloned,omitempty"`
FilesCopied uint64 `json:"files_copied,omitempty"`
TotalBytes uint64 `json:"total_bytes,omitempty"`
BytesRestored uint64 `json:"bytes_restored,omitempty"`
BytesSkipped uint64 `json:"bytes_skipped,omitempty"`
BytesCloned uint64 `json:"bytes_cloned,omitempty"`
BytesCopied uint64 `json:"bytes_copied,omitempty"`
}
type errorObject struct {
@ -141,7 +155,11 @@ type summaryOutput struct {
FilesRestored uint64 `json:"files_restored,omitempty"`
FilesSkipped uint64 `json:"files_skipped,omitempty"`
FilesDeleted uint64 `json:"files_deleted,omitempty"`
FilesCloned uint64 `json:"files_cloned,omitempty"`
FilesCopied uint64 `json:"files_copied,omitempty"`
TotalBytes uint64 `json:"total_bytes,omitempty"`
BytesRestored uint64 `json:"bytes_restored,omitempty"`
BytesSkipped uint64 `json:"bytes_skipped,omitempty"`
BytesCloned uint64 `json:"bytes_cloned,omitempty"`
BytesCopied uint64 `json:"bytes_copied,omitempty"`
}

10
internal/ui/restore/json_test.go
View file

@ -17,31 +17,31 @@ func createJSONProgress() (*ui.MockTerminal, ProgressPrinter) {
func TestJSONPrintUpdate(t *testing.T) {
term, printer := createJSONProgress()
printer.Update(State{3, 11, 0, 0, 29, 47, 0}, 5*time.Second)
printer.Update(State{3, 11, 0, 0, 0, 0, 29, 47, 0, 0, 0}, 5*time.Second)
test.Equals(t, []string{"{\"message_type\":\"status\",\"seconds_elapsed\":5,\"percent_done\":0.6170212765957447,\"total_files\":11,\"files_restored\":3,\"total_bytes\":47,\"bytes_restored\":29}\n"}, term.Output)
}
func TestJSONPrintUpdateWithSkipped(t *testing.T) {
term, printer := createJSONProgress()
printer.Update(State{3, 11, 2, 0, 29, 47, 59}, 5*time.Second)
printer.Update(State{3, 11, 2, 0, 0, 0, 29, 47, 59, 0, 0}, 5*time.Second)
test.Equals(t, []string{"{\"message_type\":\"status\",\"seconds_elapsed\":5,\"percent_done\":0.6170212765957447,\"total_files\":11,\"files_restored\":3,\"files_skipped\":2,\"total_bytes\":47,\"bytes_restored\":29,\"bytes_skipped\":59}\n"}, term.Output)
}
func TestJSONPrintSummaryOnSuccess(t *testing.T) {
term, printer := createJSONProgress()
printer.Finish(State{11, 11, 0, 0, 47, 47, 0}, 5*time.Second)
printer.Finish(State{11, 11, 0, 0, 0, 0, 47, 47, 0, 0, 0}, 5*time.Second)
test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":11,\"total_bytes\":47,\"bytes_restored\":47}\n"}, term.Output)
}
func TestJSONPrintSummaryOnErrors(t *testing.T) {
term, printer := createJSONProgress()
printer.Finish(State{3, 11, 0, 0, 29, 47, 0}, 5*time.Second)
printer.Finish(State{3, 11, 0, 0, 0, 0, 29, 47, 0, 0, 0}, 5*time.Second)
test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":3,\"total_bytes\":47,\"bytes_restored\":29}\n"}, term.Output)
}
func TestJSONPrintSummaryOnSuccessWithSkipped(t *testing.T) {
term, printer := createJSONProgress()
printer.Finish(State{11, 11, 2, 0, 47, 47, 59}, 5*time.Second)
printer.Finish(State{11, 11, 2, 0, 0, 0, 47, 47, 59, 0, 0}, 5*time.Second)
test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":11,\"files_skipped\":2,\"total_bytes\":47,\"bytes_restored\":47,\"bytes_skipped\":59}\n"}, term.Output)
}

31
internal/ui/restore/progress.go
View file

@ -12,9 +12,13 @@ type State struct {
FilesTotal uint64
FilesSkipped uint64
FilesDeleted uint64
FilesCloned uint64
FilesCopied uint64
AllBytesWritten uint64
AllBytesTotal uint64
AllBytesSkipped uint64
AllBytesCloned uint64
AllBytesCopied uint64
}
type Progress struct {
@ -47,6 +51,7 @@ const (
ActionDirRestored ItemAction = "dir restored"
ActionFileRestored ItemAction = "file restored"
ActionFileUpdated ItemAction = "file updated"
ActionFileCloned ItemAction = "file cloned"
ActionFileUnchanged ItemAction = "file unchanged"
ActionOtherRestored ItemAction = "other restored"
ActionDeleted ItemAction = "deleted"
@ -111,6 +116,32 @@ func (p *Progress) AddProgress(name string, action ItemAction, bytesWrittenPorti
}
}
// AddClonedFile records progress for a locally copied/cloned file. It is assumed
// that a file is never partially copied/cloned. The blockCloned flag describes
// whether the file was cloned using filesystem-specific block cloning facilities
// (i.e. "reflinks", leading to storage deduplication), or merely copied.
func (p *Progress) AddClonedFile(name string, size uint64, blockCloned bool) {
if p == nil {
return
}
p.m.Lock()
defer p.m.Unlock()
p.s.AllBytesWritten += size
p.s.FilesFinished++
if blockCloned {
p.s.AllBytesCloned += size
p.s.FilesCloned++
} else {
p.s.AllBytesCopied += size
p.s.FilesCopied++
}
p.printer.CompleteItem(ActionFileCloned, name, size)
}
func (p *Progress) AddSkippedFile(name string, size uint64) {
if p == nil {
return

16
internal/ui/restore/progress_test.go
View file

@ -72,7 +72,7 @@ func TestNew(t *testing.T) {
return false
})
test.Equals(t, printerTrace{
printerTraceEntry{State{0, 0, 0, 0, 0, 0, 0}, 0, false},
printerTraceEntry{State{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 0, false},
}, result)
test.Equals(t, itemTrace{}, items)
}
@ -85,7 +85,7 @@ func TestAddFile(t *testing.T) {
return false
})
test.Equals(t, printerTrace{
printerTraceEntry{State{0, 1, 0, 0, 0, fileSize, 0}, 0, false},
printerTraceEntry{State{0, 1, 0, 0, 0, 0, 0, fileSize, 0, 0, 0}, 0, false},
}, result)
test.Equals(t, itemTrace{}, items)
}
@ -100,7 +100,7 @@ func TestFirstProgressOnAFile(t *testing.T) {
return false
})
test.Equals(t, printerTrace{
printerTraceEntry{State{0, 1, 0, 0, expectedBytesWritten, expectedBytesTotal, 0}, 0, false},
printerTraceEntry{State{0, 1, 0, 0, 0, 0, expectedBytesWritten, expectedBytesTotal, 0, 0, 0}, 0, false},
}, result)
test.Equals(t, itemTrace{}, items)
}
@ -116,7 +116,7 @@ func TestLastProgressOnAFile(t *testing.T) {
return false
})
test.Equals(t, printerTrace{
printerTraceEntry{State{1, 1, 0, 0, fileSize, fileSize, 0}, 0, false},
printerTraceEntry{State{1, 1, 0, 0, 0, 0, fileSize, fileSize, 0, 0, 0}, 0, false},
}, result)
test.Equals(t, itemTrace{
itemTraceEntry{action: ActionFileUpdated, item: "test", size: fileSize},
@ -135,7 +135,7 @@ func TestLastProgressOnLastFile(t *testing.T) {
return false
})
test.Equals(t, printerTrace{
printerTraceEntry{State{2, 2, 0, 0, 50 + fileSize, 50 + fileSize, 0}, 0, false},
printerTraceEntry{State{2, 2, 0, 0, 0, 0, 50 + fileSize, 50 + fileSize, 0, 0, 0}, 0, false},
}, result)
test.Equals(t, itemTrace{
itemTraceEntry{action: ActionFileUpdated, item: "test1", size: 50},
@ -154,7 +154,7 @@ func TestSummaryOnSuccess(t *testing.T) {
return true
})
test.Equals(t, printerTrace{
printerTraceEntry{State{2, 2, 0, 0, 50 + fileSize, 50 + fileSize, 0}, mockFinishDuration, true},
printerTraceEntry{State{2, 2, 0, 0, 0, 0, 50 + fileSize, 50 + fileSize, 0, 0, 0}, mockFinishDuration, true},
}, result)
}
@ -169,7 +169,7 @@ func TestSummaryOnErrors(t *testing.T) {
return true
})
test.Equals(t, printerTrace{
printerTraceEntry{State{1, 2, 0, 0, 50 + fileSize/2, 50 + fileSize, 0}, mockFinishDuration, true},
printerTraceEntry{State{1, 2, 0, 0, 0, 0, 50 + fileSize/2, 50 + fileSize, 0, 0, 0}, mockFinishDuration, true},
}, result)
}
@ -181,7 +181,7 @@ func TestSkipFile(t *testing.T) {
return true
})
test.Equals(t, printerTrace{
printerTraceEntry{State{0, 0, 1, 0, 0, 0, fileSize}, mockFinishDuration, true},
printerTraceEntry{State{0, 0, 1, 0, 0, 0, 0, 0, fileSize, 0, 0}, mockFinishDuration, true},
}, result)
test.Equals(t, itemTrace{
itemTraceEntry{ActionFileUnchanged, "test", fileSize},

14
internal/ui/restore/text.go
View file

@ -33,6 +33,12 @@ func (t *textPrinter) Update(p State, duration time.Duration) {
if p.FilesDeleted > 0 {
progress += fmt.Sprintf(", deleted %v files/dirs", p.FilesDeleted)
}
if p.FilesCloned > 0 {
progress += fmt.Sprintf(", cloned %v files %s", p.FilesCloned, ui.FormatBytes(p.AllBytesCloned))
}
if p.FilesCopied > 0 {
progress += fmt.Sprintf(", copied %v files %s", p.FilesCopied, ui.FormatBytes(p.AllBytesCopied))
}
t.terminal.SetStatus([]string{progress})
}
@ -51,6 +57,8 @@ func (t *textPrinter) CompleteItem(messageType ItemAction, item string, size uin
action = "restored"
case ActionOtherRestored:
action = "restored"
case ActionFileCloned:
action = "cloned"
case ActionFileUpdated:
action = "updated"
case ActionFileUnchanged:
@ -88,6 +96,12 @@ func (t *textPrinter) Finish(p State, duration time.Duration) {
if p.FilesDeleted > 0 {
summary += fmt.Sprintf(", deleted %v files/dirs", p.FilesDeleted)
}
if p.FilesCloned > 0 {
summary += fmt.Sprintf(", cloned %v files (%s)", p.FilesCloned, ui.FormatBytes(p.AllBytesCloned))
}
if p.FilesCopied > 0 {
summary += fmt.Sprintf(", copied %v files (%s)", p.FilesCopied, ui.FormatBytes(p.AllBytesCopied))
}
t.terminal.Print(summary)
}

10
internal/ui/restore/text_test.go
View file

@ -17,31 +17,31 @@ func createTextProgress() (*ui.MockTerminal, ProgressPrinter) {
func TestPrintUpdate(t *testing.T) {
term, printer := createTextProgress()
printer.Update(State{3, 11, 0, 0, 29, 47, 0}, 5*time.Second)
printer.Update(State{3, 11, 0, 0, 0, 0, 29, 47, 0, 0, 0}, 5*time.Second)
test.Equals(t, []string{"[0:05] 61.70% 3 files/dirs 29 B, total 11 files/dirs 47 B"}, term.Output)
}
func TestPrintUpdateWithSkipped(t *testing.T) {
term, printer := createTextProgress()
printer.Update(State{3, 11, 2, 0, 29, 47, 59}, 5*time.Second)
printer.Update(State{3, 11, 2, 0, 0, 0, 29, 47, 59, 0, 0}, 5*time.Second)
test.Equals(t, []string{"[0:05] 61.70% 3 files/dirs 29 B, total 11 files/dirs 47 B, skipped 2 files/dirs 59 B"}, term.Output)
}
func TestPrintSummaryOnSuccess(t *testing.T) {
term, printer := createTextProgress()
printer.Finish(State{11, 11, 0, 0, 47, 47, 0}, 5*time.Second)
printer.Finish(State{11, 11, 0, 0, 0, 0, 47, 47, 0, 0, 0}, 5*time.Second)
test.Equals(t, []string{"Summary: Restored 11 files/dirs (47 B) in 0:05"}, term.Output)
}
func TestPrintSummaryOnErrors(t *testing.T) {
term, printer := createTextProgress()
printer.Finish(State{3, 11, 0, 0, 29, 47, 0}, 5*time.Second)
printer.Finish(State{3, 11, 0, 0, 0, 0, 29, 47, 0, 0, 0}, 5*time.Second)
test.Equals(t, []string{"Summary: Restored 3 / 11 files/dirs (29 B / 47 B) in 0:05"}, term.Output)
}
func TestPrintSummaryOnSuccessWithSkipped(t *testing.T) {
term, printer := createTextProgress()
printer.Finish(State{11, 11, 2, 0, 47, 47, 59}, 5*time.Second)
printer.Finish(State{11, 11, 2, 0, 0, 0, 47, 47, 59, 0, 0}, 5*time.Second)
test.Equals(t, []string{"Summary: Restored 11 files/dirs (47 B) in 0:05, skipped 2 files/dirs (59 B)"}, term.Output)
}