From ffca60231570f7a23f8304011a98c85926117433 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 11 Sep 2022 13:51:08 +0200 Subject: [PATCH 01/90] repository: Fix panic in benchmarkLoadIndex --- internal/repository/repository_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index f3516856e..f26bf46f2 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -346,6 +346,7 @@ func benchmarkLoadIndex(b *testing.B, version uint) { }, }) } + idx.Finalize() id, err := index.SaveIndex(context.TODO(), repo, idx) rtest.OK(b, err) From 0c1240360dbf7ff04423ff3d2dfcabf7600a290e Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 22 Oct 2022 23:37:31 +0200 Subject: [PATCH 02/90] index: add garbage collection benchmark Allocates an index and repeatedly triggers the GC. --- internal/index/master_index_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/internal/index/master_index_test.go b/internal/index/master_index_test.go index 9a1970827..5d12956bd 100644 --- a/internal/index/master_index_test.go +++ b/internal/index/master_index_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math/rand" + "runtime" "testing" "time" @@ -323,6 +324,17 @@ func BenchmarkMasterIndexEach(b *testing.B) { } } +func BenchmarkMasterIndexGC(b *testing.B) { + mIdx, _ := createRandomMasterIndex(b, rand.New(rand.NewSource(0)), 100, 10000) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + runtime.GC() + } + runtime.KeepAlive(mIdx) +} + var ( snapshotTime = time.Unix(1470492820, 207401672) depth = 3 From b217f38ee70d064b18c10378eccfc190c254743d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 5 Feb 2022 21:25:23 +0100 Subject: [PATCH 03/90] index: Remove pointers from within indexentrys The indexEntry objects are now allocated in a separate array. References to an indexEntry are now stored as array indices. This has the benefit of allowing the garbage collector to ignore the indexEntry objects as these do not contain pointers and are part of a single large allocation. --- internal/index/indexmap.go | 78 +++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 43 deletions(-) diff --git a/internal/index/indexmap.go b/internal/index/indexmap.go index ef3539d48..60ab11ff7 100644 --- a/internal/index/indexmap.go +++ b/internal/index/indexmap.go @@ -17,12 +17,12 @@ import ( // needs to be resized when the table grows, preventing memory usage spikes. type indexMap struct { // The number of buckets is always a power of two and never zero. - buckets []*indexEntry + buckets []uint numentries uint mh maphash.Hash - free *indexEntry // Free list. + blockList []indexEntry } const ( @@ -41,7 +41,7 @@ func (m *indexMap) add(id restic.ID, packIdx int, offset, length uint32, uncompr } h := m.hash(id) - e := m.newEntry() + e, idx := m.newEntry() e.id = id e.next = m.buckets[h] // Prepend to existing chain. e.packIndex = packIdx @@ -49,18 +49,19 @@ func (m *indexMap) add(id restic.ID, packIdx int, offset, length uint32, uncompr e.length = length e.uncompressedLength = uncompressedLength - m.buckets[h] = e + m.buckets[h] = idx m.numentries++ } // foreach calls fn for all entries in the map, until fn returns false. func (m *indexMap) foreach(fn func(*indexEntry) bool) { - for _, e := range m.buckets { - for e != nil { + for _, ei := range m.buckets { + for ei != 0 { + e := m.resolve(ei) if !fn(e) { return } - e = e.next + ei = e.next } } } @@ -72,7 +73,10 @@ func (m *indexMap) foreachWithID(id restic.ID, fn func(*indexEntry)) { } h := m.hash(id) - for e := m.buckets[h]; e != nil; e = e.next { + ei := m.buckets[h] + for ei != 0 { + e := m.resolve(ei) + ei = e.next if e.id != id { continue } @@ -87,25 +91,29 @@ func (m *indexMap) get(id restic.ID) *indexEntry { } h := m.hash(id) - for e := m.buckets[h]; e != nil; e = e.next { + ei := m.buckets[h] + for ei != 0 { + e := m.resolve(ei) if e.id == id { return e } + ei = e.next } return nil } func (m *indexMap) grow() { old := m.buckets - m.buckets = make([]*indexEntry, growthFactor*len(m.buckets)) + m.buckets = make([]uint, growthFactor*len(m.buckets)) - for _, e := range old { - for e != nil { + for _, ei := range old { + for ei != 0 { + e := m.resolve(ei) h := m.hash(e.id) next := e.next e.next = m.buckets[h] - m.buckets[h] = e - e = next + m.buckets[h] = ei + ei = next } } } @@ -124,45 +132,29 @@ func (m *indexMap) hash(id restic.ID) uint { func (m *indexMap) init() { const initialBuckets = 64 - m.buckets = make([]*indexEntry, initialBuckets) + m.buckets = make([]uint, initialBuckets) + // first entry in blockList serves as null byte + m.blockList = make([]indexEntry, 1) } func (m *indexMap) len() uint { return m.numentries } -func (m *indexMap) newEntry() *indexEntry { - // We keep a free list of objects to speed up allocation and GC. - // There's an obvious trade-off here: allocating in larger batches - // means we allocate faster and the GC has to keep fewer bits to track - // what we have in use, but it means we waste some space. - // - // Then again, allocating each indexEntry separately also wastes space - // on 32-bit platforms, because the Go malloc has no size class for - // exactly 52 bytes, so it puts the indexEntry in a 64-byte slot instead. - // See src/runtime/sizeclasses.go in the Go source repo. - // - // The batch size of 4 means we hit the size classes for 4×64=256 bytes - // (64-bit) and 4×52=208 bytes (32-bit), wasting nothing in malloc on - // 64-bit and relatively little on 32-bit. - const entryAllocBatch = 4 +func (m *indexMap) newEntry() (*indexEntry, uint) { + m.blockList = append(m.blockList, indexEntry{}) - e := m.free - if e != nil { - m.free = e.next - } else { - free := new([entryAllocBatch]indexEntry) - e = &free[0] - for i := 1; i < len(free)-1; i++ { - free[i].next = &free[i+1] - } - m.free = &free[1] - } + idx := uint(len(m.blockList) - 1) + e := &m.blockList[idx] - return e + return e, idx +} + +func (m *indexMap) resolve(idx uint) *indexEntry { + return &m.blockList[idx] } type indexEntry struct { id restic.ID - next *indexEntry + next uint packIndex int // Position in containing Index's packs field. offset uint32 length uint32 From fed33295c38197cec49d164725e744c1c9786690 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 28 May 2023 23:42:47 +0200 Subject: [PATCH 04/90] index: store indexEntries in hashed array tree This data structure reduces the wasted memory to O(sqrt(n)). The top-layer of the hashed array tree (HAT) also has a size of O(sqrt(n)), which makes it cache efficient. The top-layer should be small enough to easily fit into the CPU cache and thus only adds little overhead compared to directly accessing an index entry via a pointer. --- internal/index/indexmap.go | 94 ++++++++++++++++++++++++++++++++++---- 1 file changed, 85 insertions(+), 9 deletions(-) diff --git a/internal/index/indexmap.go b/internal/index/indexmap.go index 60ab11ff7..811d20903 100644 --- a/internal/index/indexmap.go +++ b/internal/index/indexmap.go @@ -1,6 +1,7 @@ package index import ( + "fmt" "hash/maphash" "github.com/restic/restic/internal/restic" @@ -22,7 +23,7 @@ type indexMap struct { mh maphash.Hash - blockList []indexEntry + blockList hashedArrayTree } const ( @@ -134,22 +135,18 @@ func (m *indexMap) init() { const initialBuckets = 64 m.buckets = make([]uint, initialBuckets) // first entry in blockList serves as null byte - m.blockList = make([]indexEntry, 1) + m.blockList = *newHAT() + m.newEntry() } func (m *indexMap) len() uint { return m.numentries } func (m *indexMap) newEntry() (*indexEntry, uint) { - m.blockList = append(m.blockList, indexEntry{}) - - idx := uint(len(m.blockList) - 1) - e := &m.blockList[idx] - - return e, idx + return m.blockList.Alloc() } func (m *indexMap) resolve(idx uint) *indexEntry { - return &m.blockList[idx] + return m.blockList.Ref(idx) } type indexEntry struct { @@ -160,3 +157,82 @@ type indexEntry struct { length uint32 uncompressedLength uint32 } + +type hashedArrayTree struct { + mask uint + maskShift uint + blockSize uint + + size uint + blockList [][]indexEntry +} + +func newHAT() *hashedArrayTree { + // start with a small block size + blockSizePower := uint(2) + blockSize := uint(1 << blockSizePower) + + return &hashedArrayTree{ + mask: blockSize - 1, + maskShift: blockSizePower, + blockSize: blockSize, + size: 0, + blockList: make([][]indexEntry, blockSize), + } +} + +func (h *hashedArrayTree) Alloc() (*indexEntry, uint) { + h.grow() + size := h.size + idx, subIdx := h.index(size) + h.size++ + return &h.blockList[idx][subIdx], size +} + +func (h *hashedArrayTree) index(pos uint) (idx uint, subIdx uint) { + subIdx = pos & h.mask + idx = pos >> h.maskShift + return +} + +func (h *hashedArrayTree) Ref(pos uint) *indexEntry { + if pos >= h.size { + panic(fmt.Sprintf("array index %d out of bounds %d", pos, h.size)) + } + + idx, subIdx := h.index(pos) + return &h.blockList[idx][subIdx] +} + +func (h *hashedArrayTree) Size() uint { + return h.size +} + +func (h *hashedArrayTree) grow() { + idx, subIdx := h.index(h.size) + if int(idx) == len(h.blockList) { + // blockList is too small -> double list and block size + oldBlocks := h.blockList + h.blockList = make([][]indexEntry, h.blockSize) + + h.blockSize *= 2 + h.mask = h.mask*2 + 1 + h.maskShift++ + idx = idx / 2 + + // pairwise merging of blocks + for i := 0; i < len(oldBlocks); i += 2 { + block := make([]indexEntry, 0, h.blockSize) + block = append(block, oldBlocks[i]...) + block = append(block, oldBlocks[i+1]...) + h.blockList[i/2] = block + // allow GC + oldBlocks[i] = nil + oldBlocks[i+1] = nil + } + } + if subIdx == 0 { + // new index entry batch + h.blockList[idx] = make([]indexEntry, h.blockSize) + } +} From 12141afbada61d03495ef2830732e4d2c406fce4 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 29 May 2023 01:09:33 +0200 Subject: [PATCH 05/90] index: Allow inlining of HAT --- internal/index/indexmap.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/index/indexmap.go b/internal/index/indexmap.go index 811d20903..121ed09af 100644 --- a/internal/index/indexmap.go +++ b/internal/index/indexmap.go @@ -1,7 +1,6 @@ package index import ( - "fmt" "hash/maphash" "github.com/restic/restic/internal/restic" @@ -197,7 +196,7 @@ func (h *hashedArrayTree) index(pos uint) (idx uint, subIdx uint) { func (h *hashedArrayTree) Ref(pos uint) *indexEntry { if pos >= h.size { - panic(fmt.Sprintf("array index %d out of bounds %d", pos, h.size)) + panic("array index out of bounds") } idx, subIdx := h.index(pos) From f1c388c623fee735871067b912e6f5632f33f772 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 29 May 2023 00:13:32 +0200 Subject: [PATCH 06/90] index: remove redundant storage of indexmap size --- internal/index/indexmap.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/internal/index/indexmap.go b/internal/index/indexmap.go index 121ed09af..dfb4a1422 100644 --- a/internal/index/indexmap.go +++ b/internal/index/indexmap.go @@ -2,6 +2,7 @@ package index import ( "hash/maphash" + "math" "github.com/restic/restic/internal/restic" ) @@ -17,8 +18,7 @@ import ( // needs to be resized when the table grows, preventing memory usage spikes. type indexMap struct { // The number of buckets is always a power of two and never zero. - buckets []uint - numentries uint + buckets []uint mh maphash.Hash @@ -34,9 +34,9 @@ const ( // using id as the key. func (m *indexMap) add(id restic.ID, packIdx int, offset, length uint32, uncompressedLength uint32) { switch { - case m.numentries == 0: // Lazy initialization. + case m.len() == math.MaxUint: // Lazy initialization. m.init() - case m.numentries >= maxLoad*uint(len(m.buckets)): + case m.len() >= maxLoad*uint(len(m.buckets)): m.grow() } @@ -50,7 +50,6 @@ func (m *indexMap) add(id restic.ID, packIdx int, offset, length uint32, uncompr e.uncompressedLength = uncompressedLength m.buckets[h] = idx - m.numentries++ } // foreach calls fn for all entries in the map, until fn returns false. @@ -138,7 +137,7 @@ func (m *indexMap) init() { m.newEntry() } -func (m *indexMap) len() uint { return m.numentries } +func (m *indexMap) len() uint { return m.blockList.Size() - 1 } func (m *indexMap) newEntry() (*indexEntry, uint) { return m.blockList.Alloc() From fc05e35a08c689df0d03fc7d4bcb5bb13b0548b1 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Tue, 30 May 2023 20:12:36 +0200 Subject: [PATCH 07/90] index: let indexmap.Each iterate in allocation order Iterating through the indexmap according to the bucket order has the problem that all indexEntries are accessed in random order which is rather cache inefficient. As we already keep a list of all allocated blocks, just iterate through it. This allows iterating through a batch of indexEntries without random memory accesses. In addition, the packID will likely remain similar across multiple blobs as all blobs of a pack file are added as a single batch. --- internal/index/indexmap.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/internal/index/indexmap.go b/internal/index/indexmap.go index dfb4a1422..15f253d76 100644 --- a/internal/index/indexmap.go +++ b/internal/index/indexmap.go @@ -54,13 +54,10 @@ func (m *indexMap) add(id restic.ID, packIdx int, offset, length uint32, uncompr // foreach calls fn for all entries in the map, until fn returns false. func (m *indexMap) foreach(fn func(*indexEntry) bool) { - for _, ei := range m.buckets { - for ei != 0 { - e := m.resolve(ei) - if !fn(e) { - return - } - ei = e.next + blockCount := m.blockList.Size() + for i := uint(1); i < blockCount; i++ { + if !fn(m.resolve(i)) { + return } } } From 9a7056a4790ea27c109d6594cbf38b1e34ad4873 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Tue, 30 May 2023 20:13:33 +0200 Subject: [PATCH 08/90] index: implement indexmap.grow() without random access --- internal/index/indexmap.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/internal/index/indexmap.go b/internal/index/indexmap.go index 15f253d76..c709f7b3a 100644 --- a/internal/index/indexmap.go +++ b/internal/index/indexmap.go @@ -99,18 +99,15 @@ func (m *indexMap) get(id restic.ID) *indexEntry { } func (m *indexMap) grow() { - old := m.buckets m.buckets = make([]uint, growthFactor*len(m.buckets)) - for _, ei := range old { - for ei != 0 { - e := m.resolve(ei) - h := m.hash(e.id) - next := e.next - e.next = m.buckets[h] - m.buckets[h] = ei - ei = next - } + blockCount := m.blockList.Size() + for i := uint(1); i < blockCount; i++ { + e := m.resolve(i) + + h := m.hash(e.id) + e.next = m.buckets[h] + m.buckets[h] = i } } From ac1dfc99bb01205dab5d99abcea8c58e3439975e Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 2 Jun 2023 19:39:12 +0200 Subject: [PATCH 09/90] index: fix blocklist size --- internal/index/indexmap.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/index/indexmap.go b/internal/index/indexmap.go index c709f7b3a..b779b0527 100644 --- a/internal/index/indexmap.go +++ b/internal/index/indexmap.go @@ -204,14 +204,14 @@ func (h *hashedArrayTree) grow() { idx, subIdx := h.index(h.size) if int(idx) == len(h.blockList) { // blockList is too small -> double list and block size - oldBlocks := h.blockList - h.blockList = make([][]indexEntry, h.blockSize) - h.blockSize *= 2 h.mask = h.mask*2 + 1 h.maskShift++ idx = idx / 2 + oldBlocks := h.blockList + h.blockList = make([][]indexEntry, h.blockSize) + // pairwise merging of blocks for i := 0; i < len(oldBlocks); i += 2 { block := make([]indexEntry, 0, h.blockSize) From f1b73c9301bd35388f058dcd202358ce1c85a40b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 2 Jun 2023 21:51:50 +0200 Subject: [PATCH 10/90] Reduce GOGC to 50 The index used by restic consumes a major part of the total memory usage. This leads to an unnecessarily large amount of memory that contains ephemeral objects that are only used for a short time. --- cmd/restic/main.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cmd/restic/main.go b/cmd/restic/main.go index 392177d13..64b75b43a 100644 --- a/cmd/restic/main.go +++ b/cmd/restic/main.go @@ -7,6 +7,7 @@ import ( "log" "os" "runtime" + godebug "runtime/debug" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/options" @@ -81,7 +82,16 @@ func needsPassword(cmd string) bool { var logBuffer = bytes.NewBuffer(nil) +func tweakGoGC() { + // lower GOGC from 100 to 50, unless it was manually overwritten by the user + oldValue := godebug.SetGCPercent(50) + if oldValue != 100 { + godebug.SetGCPercent(oldValue) + } +} + func main() { + tweakGoGC() // install custom global logger into a buffer, if an error occurs // we can show the logs log.SetOutput(logBuffer) From eef0ee7a85994e4696368e39d8f7d13e07017115 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 2 Jun 2023 21:56:14 +0200 Subject: [PATCH 11/90] repository: trigger GC after loading the index Loading the index requires some scratch space, thus make sure that this memory does not factor into the targeted gc memory usage limit. --- internal/repository/repository.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 9d1b40c64..653c1f774 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "os" + "runtime" "sort" "sync" @@ -601,6 +602,9 @@ func (r *Repository) LoadIndex(ctx context.Context) error { return err } + // Trigger GC to reset garbage collection threshold + runtime.GC() + if r.cfg.Version < 2 { // sanity check ctx, cancel := context.WithCancel(ctx) From 2fcb3947df77427cafdb613acd831d9cbc111530 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 2 Jun 2023 21:57:40 +0200 Subject: [PATCH 12/90] prune: trigger GC after prune planning --- cmd/restic/cmd_prune.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index 26f21b1f3..1889dffd6 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -3,6 +3,7 @@ package main import ( "context" "math" + "runtime" "sort" "strconv" "strings" @@ -205,6 +206,9 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption return err } + // Trigger GC to reset garbage collection threshold + runtime.GC() + return doPrune(ctx, opts, gopts, repo, plan) } From aea7538936316dde8a615b4751d65d873db10ff1 Mon Sep 17 00:00:00 2001 From: Refutable4890 <107740882+Refutable4890@users.noreply.github.com> Date: Wed, 7 Jun 2023 22:35:33 +0800 Subject: [PATCH 13/90] Add an description of `check` temporary cache directory location --- doc/manual_rest.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index 093144722..b83a152bf 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -418,6 +418,7 @@ instead of the default, set the environment variable like this: $ restic -r /srv/restic-repo backup ~/work +If the environment variable ``$RESTIC_CACHE_DIR`` is not set, ``check`` creates its temporary cache directory in the temporary directory. If ``$RESTIC_CACHE_DIR`` is set, ``check`` creates its temporary cache directory in ``$RESTIC_CACHE_DIR``. .. _caching: From 0f80b6a137e0ffd79dce0887be457d62c0c826db Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 8 Jun 2023 18:02:46 +0200 Subject: [PATCH 14/90] add changelog for gc tuning --- changelog/unreleased/issue-3328 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/issue-3328 diff --git a/changelog/unreleased/issue-3328 b/changelog/unreleased/issue-3328 new file mode 100644 index 000000000..a8ef76d79 --- /dev/null +++ b/changelog/unreleased/issue-3328 @@ -0,0 +1,5 @@ +Enhancement: Reduce memory usage by up to 25% + +https://github.com/restic/restic/issues/3328 +https://github.com/restic/restic/pull/4352 +https://github.com/restic/restic/pull/4353 From 55c21846b1d7a6d62126340bfc2aaef5f10981ca Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 8 Jun 2023 18:07:06 +0200 Subject: [PATCH 15/90] Revert "index: remove redundant storage of indexmap size" This reverts commit f1c388c623fee735871067b912e6f5632f33f772. For an uninitialized indexmap the returned size was `-1` which is unexpected and could cause problems. --- internal/index/indexmap.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/internal/index/indexmap.go b/internal/index/indexmap.go index b779b0527..2386e01b6 100644 --- a/internal/index/indexmap.go +++ b/internal/index/indexmap.go @@ -2,7 +2,6 @@ package index import ( "hash/maphash" - "math" "github.com/restic/restic/internal/restic" ) @@ -18,7 +17,8 @@ import ( // needs to be resized when the table grows, preventing memory usage spikes. type indexMap struct { // The number of buckets is always a power of two and never zero. - buckets []uint + buckets []uint + numentries uint mh maphash.Hash @@ -34,9 +34,9 @@ const ( // using id as the key. func (m *indexMap) add(id restic.ID, packIdx int, offset, length uint32, uncompressedLength uint32) { switch { - case m.len() == math.MaxUint: // Lazy initialization. + case m.numentries == 0: // Lazy initialization. m.init() - case m.len() >= maxLoad*uint(len(m.buckets)): + case m.numentries >= maxLoad*uint(len(m.buckets)): m.grow() } @@ -50,6 +50,7 @@ func (m *indexMap) add(id restic.ID, packIdx int, offset, length uint32, uncompr e.uncompressedLength = uncompressedLength m.buckets[h] = idx + m.numentries++ } // foreach calls fn for all entries in the map, until fn returns false. @@ -131,7 +132,7 @@ func (m *indexMap) init() { m.newEntry() } -func (m *indexMap) len() uint { return m.blockList.Size() - 1 } +func (m *indexMap) len() uint { return m.numentries } func (m *indexMap) newEntry() (*indexEntry, uint) { return m.blockList.Alloc() From bb20078641e79e5b8eb8cf4025b05e40a646620d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 1 May 2023 12:51:37 +0200 Subject: [PATCH 16/90] restore: pass termStatus to restore in tests --- cmd/restic/cmd_backup_integration_test.go | 33 ++++++---------------- cmd/restic/cmd_restore_integration_test.go | 11 +++++--- cmd/restic/integration_helpers_test.go | 19 +++++++++++++ 3 files changed, 35 insertions(+), 28 deletions(-) diff --git a/cmd/restic/cmd_backup_integration_test.go b/cmd/restic/cmd_backup_integration_test.go index 3af16a2be..b81db21e6 100644 --- a/cmd/restic/cmd_backup_integration_test.go +++ b/cmd/restic/cmd_backup_integration_test.go @@ -13,34 +13,19 @@ import ( "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" "github.com/restic/restic/internal/ui/termstatus" - "golang.org/x/sync/errgroup" ) func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) error { - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() + return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error { + t.Logf("backing up %v in %v", target, dir) + if dir != "" { + cleanup := rtest.Chdir(t, dir) + defer cleanup() + } - var wg errgroup.Group - term := termstatus.New(gopts.stdout, gopts.stderr, gopts.Quiet) - wg.Go(func() error { term.Run(ctx); return nil }) - - t.Logf("backing up %v in %v", target, dir) - if dir != "" { - cleanup := rtest.Chdir(t, dir) - defer cleanup() - } - - opts.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true} - backupErr := runBackup(ctx, opts, gopts, term, target) - - cancel() - - err := wg.Wait() - if err != nil { - t.Fatal(err) - } - - return backupErr + opts.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true} + return runBackup(ctx, opts, gopts, term, target) + }) } func testRunBackup(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) { diff --git a/cmd/restic/cmd_restore_integration_test.go b/cmd/restic/cmd_restore_integration_test.go index 74fddd347..2c7cbe1fb 100644 --- a/cmd/restic/cmd_restore_integration_test.go +++ b/cmd/restic/cmd_restore_integration_test.go @@ -14,6 +14,7 @@ import ( "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui/termstatus" ) func testRunRestore(t testing.TB, opts GlobalOptions, dir string, snapshotID restic.ID) { @@ -26,11 +27,13 @@ func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snaps Exclude: excludes, } - rtest.OK(t, runRestore(context.TODO(), opts, gopts, nil, []string{snapshotID.String()})) + rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts)) } func testRunRestoreAssumeFailure(snapshotID string, opts RestoreOptions, gopts GlobalOptions) error { - return runRestore(context.TODO(), opts, gopts, nil, []string{snapshotID}) + return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error { + return runRestore(ctx, opts, gopts, term, []string{snapshotID}) + }) } func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths []string, hosts []string) { @@ -42,7 +45,7 @@ func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths [ }, } - rtest.OK(t, runRestore(context.TODO(), opts, gopts, nil, []string{"latest"})) + rtest.OK(t, testRunRestoreAssumeFailure("latest", opts, gopts)) } func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includes []string) { @@ -51,7 +54,7 @@ func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snaps Include: includes, } - rtest.OK(t, runRestore(context.TODO(), opts, gopts, nil, []string{snapshotID.String()})) + rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts)) } func TestRestoreFilter(t *testing.T) { diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go index cdafd8c98..a0e4d49d6 100644 --- a/cmd/restic/integration_helpers_test.go +++ b/cmd/restic/integration_helpers_test.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "runtime" + "sync" "testing" "github.com/restic/restic/internal/backend/retry" @@ -17,6 +18,7 @@ import ( "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui/termstatus" ) type dirEntry struct { @@ -356,3 +358,20 @@ func withCaptureStdout(inner func() error) (*bytes.Buffer, error) { return buf, err } + +func withTermStatus(gopts GlobalOptions, callback func(ctx context.Context, term *termstatus.Terminal) error) error { + ctx, cancel := context.WithCancel(context.TODO()) + var wg sync.WaitGroup + + term := termstatus.New(gopts.stdout, gopts.stderr, gopts.Quiet) + wg.Add(1) + go func() { + defer wg.Done() + term.Run(ctx) + }() + + defer wg.Wait() + defer cancel() + + return callback(ctx, term) +} From a9aff885d69a3f9918239dd65ec83d8566af4ac3 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 1 May 2023 11:19:09 +0200 Subject: [PATCH 17/90] restore: reorganize progress bar code The structure is now much more similar to that of the backup command. --- cmd/restic/cmd_restore.go | 2 +- .../{progressformatter.go => progress.go} | 51 ++----------------- ...ressformatter_test.go => progress_test.go} | 33 ------------ internal/ui/restore/text.go | 47 +++++++++++++++++ internal/ui/restore/text_test.go | 41 +++++++++++++++ 5 files changed, 94 insertions(+), 80 deletions(-) rename internal/ui/restore/{progressformatter.go => progress.go} (58%) rename internal/ui/restore/{progressformatter_test.go => progress_test.go} (80%) create mode 100644 internal/ui/restore/text.go create mode 100644 internal/ui/restore/text_test.go diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index a0d4ce3e4..e3b46fa35 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -177,7 +177,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, var progress *restoreui.Progress if !gopts.Quiet && !gopts.JSON { - progress = restoreui.NewProgress(restoreui.NewProgressPrinter(term), calculateProgressInterval(!gopts.Quiet, gopts.JSON)) + progress = restoreui.NewProgress(restoreui.NewTextPrinter(term), calculateProgressInterval(!gopts.Quiet, gopts.JSON)) } res := restorer.NewRestorer(repo, sn, opts.Sparse, progress) diff --git a/internal/ui/restore/progressformatter.go b/internal/ui/restore/progress.go similarity index 58% rename from internal/ui/restore/progressformatter.go rename to internal/ui/restore/progress.go index a89cc628e..f2bd5d38b 100644 --- a/internal/ui/restore/progressformatter.go +++ b/internal/ui/restore/progress.go @@ -1,11 +1,9 @@ package restore import ( - "fmt" "sync" "time" - "github.com/restic/restic/internal/ui" "github.com/restic/restic/internal/ui/progress" ) @@ -28,6 +26,11 @@ type progressInfoEntry struct { bytesTotal uint64 } +type term interface { + Print(line string) + SetStatus(lines []string) +} + type ProgressPrinter interface { Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) @@ -85,47 +88,3 @@ func (p *Progress) AddProgress(name string, bytesWrittenPortion uint64, bytesTot func (p *Progress) Finish() { p.updater.Done() } - -type term interface { - Print(line string) - SetStatus(lines []string) -} - -type textPrinter struct { - terminal term -} - -func NewProgressPrinter(terminal term) ProgressPrinter { - return &textPrinter{ - terminal: terminal, - } -} - -func (t *textPrinter) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { - timeLeft := ui.FormatDuration(duration) - formattedAllBytesWritten := ui.FormatBytes(allBytesWritten) - formattedAllBytesTotal := ui.FormatBytes(allBytesTotal) - allPercent := ui.FormatPercent(allBytesWritten, allBytesTotal) - progress := fmt.Sprintf("[%s] %s %v files %s, total %v files %v", - timeLeft, allPercent, filesFinished, formattedAllBytesWritten, filesTotal, formattedAllBytesTotal) - - t.terminal.SetStatus([]string{progress}) -} - -func (t *textPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { - t.terminal.SetStatus([]string{}) - - timeLeft := ui.FormatDuration(duration) - formattedAllBytesTotal := ui.FormatBytes(allBytesTotal) - - var summary string - if filesFinished == filesTotal && allBytesWritten == allBytesTotal { - summary = fmt.Sprintf("Summary: Restored %d Files (%s) in %s", filesTotal, formattedAllBytesTotal, timeLeft) - } else { - formattedAllBytesWritten := ui.FormatBytes(allBytesWritten) - summary = fmt.Sprintf("Summary: Restored %d / %d Files (%s / %s) in %s", - filesFinished, filesTotal, formattedAllBytesWritten, formattedAllBytesTotal, timeLeft) - } - - t.terminal.Print(summary) -} diff --git a/internal/ui/restore/progressformatter_test.go b/internal/ui/restore/progress_test.go similarity index 80% rename from internal/ui/restore/progressformatter_test.go rename to internal/ui/restore/progress_test.go index e3dc3ace5..9e625aa20 100644 --- a/internal/ui/restore/progressformatter_test.go +++ b/internal/ui/restore/progress_test.go @@ -135,36 +135,3 @@ func TestSummaryOnErrors(t *testing.T) { printerTraceEntry{1, 2, 50 + fileSize/2, 50 + fileSize, mockFinishDuration, true}, }, result) } - -type mockTerm struct { - output []string -} - -func (m *mockTerm) Print(line string) { - m.output = append(m.output, line) -} - -func (m *mockTerm) SetStatus(lines []string) { - m.output = append([]string{}, lines...) -} - -func TestPrintUpdate(t *testing.T) { - term := &mockTerm{} - printer := NewProgressPrinter(term) - printer.Update(3, 11, 29, 47, 5*time.Second) - test.Equals(t, []string{"[0:05] 61.70% 3 files 29 B, total 11 files 47 B"}, term.output) -} - -func TestPrintSummaryOnSuccess(t *testing.T) { - term := &mockTerm{} - printer := NewProgressPrinter(term) - printer.Finish(11, 11, 47, 47, 5*time.Second) - test.Equals(t, []string{"Summary: Restored 11 Files (47 B) in 0:05"}, term.output) -} - -func TestPrintSummaryOnErrors(t *testing.T) { - term := &mockTerm{} - printer := NewProgressPrinter(term) - printer.Finish(3, 11, 29, 47, 5*time.Second) - test.Equals(t, []string{"Summary: Restored 3 / 11 Files (29 B / 47 B) in 0:05"}, term.output) -} diff --git a/internal/ui/restore/text.go b/internal/ui/restore/text.go new file mode 100644 index 000000000..2f73f7cfe --- /dev/null +++ b/internal/ui/restore/text.go @@ -0,0 +1,47 @@ +package restore + +import ( + "fmt" + "time" + + "github.com/restic/restic/internal/ui" +) + +type textPrinter struct { + terminal term +} + +func NewTextPrinter(terminal term) ProgressPrinter { + return &textPrinter{ + terminal: terminal, + } +} + +func (t *textPrinter) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { + timeLeft := ui.FormatDuration(duration) + formattedAllBytesWritten := ui.FormatBytes(allBytesWritten) + formattedAllBytesTotal := ui.FormatBytes(allBytesTotal) + allPercent := ui.FormatPercent(allBytesWritten, allBytesTotal) + progress := fmt.Sprintf("[%s] %s %v files %s, total %v files %v", + timeLeft, allPercent, filesFinished, formattedAllBytesWritten, filesTotal, formattedAllBytesTotal) + + t.terminal.SetStatus([]string{progress}) +} + +func (t *textPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { + t.terminal.SetStatus([]string{}) + + timeLeft := ui.FormatDuration(duration) + formattedAllBytesTotal := ui.FormatBytes(allBytesTotal) + + var summary string + if filesFinished == filesTotal && allBytesWritten == allBytesTotal { + summary = fmt.Sprintf("Summary: Restored %d Files (%s) in %s", filesTotal, formattedAllBytesTotal, timeLeft) + } else { + formattedAllBytesWritten := ui.FormatBytes(allBytesWritten) + summary = fmt.Sprintf("Summary: Restored %d / %d Files (%s / %s) in %s", + filesFinished, filesTotal, formattedAllBytesWritten, formattedAllBytesTotal, timeLeft) + } + + t.terminal.Print(summary) +} diff --git a/internal/ui/restore/text_test.go b/internal/ui/restore/text_test.go new file mode 100644 index 000000000..0dd32e686 --- /dev/null +++ b/internal/ui/restore/text_test.go @@ -0,0 +1,41 @@ +package restore + +import ( + "testing" + "time" + + "github.com/restic/restic/internal/test" +) + +type mockTerm struct { + output []string +} + +func (m *mockTerm) Print(line string) { + m.output = append(m.output, line) +} + +func (m *mockTerm) SetStatus(lines []string) { + m.output = append([]string{}, lines...) +} + +func TestPrintUpdate(t *testing.T) { + term := &mockTerm{} + printer := NewTextPrinter(term) + printer.Update(3, 11, 29, 47, 5*time.Second) + test.Equals(t, []string{"[0:05] 61.70% 3 files 29 B, total 11 files 47 B"}, term.output) +} + +func TestPrintSummaryOnSuccess(t *testing.T) { + term := &mockTerm{} + printer := NewTextPrinter(term) + printer.Finish(11, 11, 47, 47, 5*time.Second) + test.Equals(t, []string{"Summary: Restored 11 Files (47 B) in 0:05"}, term.output) +} + +func TestPrintSummaryOnErrors(t *testing.T) { + term := &mockTerm{} + printer := NewTextPrinter(term) + printer.Finish(3, 11, 29, 47, 5*time.Second) + test.Equals(t, []string{"Summary: Restored 3 / 11 Files (29 B / 47 B) in 0:05"}, term.output) +} From d54176ce5dbb3a0415924643600844b62bad7528 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 1 May 2023 12:01:03 +0200 Subject: [PATCH 18/90] restore: add basic json progress --- cmd/restic/cmd_restore.go | 28 ++++++++----- internal/ui/backup/json.go | 15 +------ internal/ui/format.go | 11 +++++ internal/ui/restore/json.go | 69 ++++++++++++++++++++++++++++++++ internal/ui/restore/text.go | 2 +- internal/ui/restore/text_test.go | 6 +-- 6 files changed, 104 insertions(+), 27 deletions(-) create mode 100644 internal/ui/restore/json.go diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index e3b46fa35..363b35370 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -175,11 +175,14 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, return err } - var progress *restoreui.Progress - if !gopts.Quiet && !gopts.JSON { - progress = restoreui.NewProgress(restoreui.NewTextPrinter(term), calculateProgressInterval(!gopts.Quiet, gopts.JSON)) + var printer restoreui.ProgressPrinter + if gopts.JSON { + printer = restoreui.NewJSONProgress(term) + } else { + printer = restoreui.NewTextProgress(term) } + progress := restoreui.NewProgress(printer, calculateProgressInterval(!gopts.Quiet, gopts.JSON)) res := restorer.NewRestorer(repo, sn, opts.Sparse, progress) totalErrors := 0 @@ -237,23 +240,25 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, res.SelectFilter = selectIncludeFilter } - Verbosef("restoring %s to %s\n", res.Snapshot(), opts.Target) + if !gopts.JSON { + Verbosef("restoring %s to %s\n", res.Snapshot(), opts.Target) + } err = res.RestoreTo(ctx, opts.Target) if err != nil { return err } - if progress != nil { - progress.Finish() - } + progress.Finish() if totalErrors > 0 { return errors.Fatalf("There were %d errors\n", totalErrors) } if opts.Verify { - Verbosef("verifying files in %s\n", opts.Target) + if !gopts.JSON { + Verbosef("verifying files in %s\n", opts.Target) + } var count int t0 := time.Now() count, err = res.VerifyFiles(ctx, opts.Target) @@ -263,8 +268,11 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, if totalErrors > 0 { return errors.Fatalf("There were %d errors\n", totalErrors) } - Verbosef("finished verifying %d files in %s (took %s)\n", count, opts.Target, - time.Since(t0).Round(time.Millisecond)) + + if !gopts.JSON { + Verbosef("finished verifying %d files in %s (took %s)\n", count, opts.Target, + time.Since(t0).Round(time.Millisecond)) + } } return nil diff --git a/internal/ui/backup/json.go b/internal/ui/backup/json.go index e7c9274a4..10f0e91fa 100644 --- a/internal/ui/backup/json.go +++ b/internal/ui/backup/json.go @@ -1,8 +1,6 @@ package backup import ( - "bytes" - "encoding/json" "sort" "time" @@ -32,21 +30,12 @@ func NewJSONProgress(term *termstatus.Terminal, verbosity uint) *JSONProgress { } } -func toJSONString(status interface{}) string { - buf := new(bytes.Buffer) - err := json.NewEncoder(buf).Encode(status) - if err != nil { - panic(err) - } - return buf.String() -} - func (b *JSONProgress) print(status interface{}) { - b.term.Print(toJSONString(status)) + b.term.Print(ui.ToJSONString(status)) } func (b *JSONProgress) error(status interface{}) { - b.term.Error(toJSONString(status)) + b.term.Error(ui.ToJSONString(status)) } // Update updates the status lines. diff --git a/internal/ui/format.go b/internal/ui/format.go index 13d02f9e3..34c97703a 100644 --- a/internal/ui/format.go +++ b/internal/ui/format.go @@ -1,6 +1,8 @@ package ui import ( + "bytes" + "encoding/json" "fmt" "time" ) @@ -53,3 +55,12 @@ func FormatSeconds(sec uint64) string { } return fmt.Sprintf("%d:%02d", min, sec) } + +func ToJSONString(status interface{}) string { + buf := new(bytes.Buffer) + err := json.NewEncoder(buf).Encode(status) + if err != nil { + panic(err) + } + return buf.String() +} diff --git a/internal/ui/restore/json.go b/internal/ui/restore/json.go new file mode 100644 index 000000000..0ff0a89cd --- /dev/null +++ b/internal/ui/restore/json.go @@ -0,0 +1,69 @@ +package restore + +import ( + "time" + + "github.com/restic/restic/internal/ui" +) + +type jsonPrinter struct { + terminal term +} + +func NewJSONProgress(terminal term) ProgressPrinter { + return &jsonPrinter{ + terminal: terminal, + } +} + +func (t *jsonPrinter) print(status interface{}) { + t.terminal.Print(ui.ToJSONString(status)) +} + +func (t *jsonPrinter) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { + status := statusUpdate{ + MessageType: "status", + SecondsElapsed: uint64(duration / time.Second), + TotalFiles: filesTotal, + FilesDone: filesFinished, + TotalBytes: allBytesTotal, + BytesDone: allBytesWritten, + } + + if allBytesTotal > 0 { + status.PercentDone = float64(allBytesWritten) / float64(allBytesTotal) + } + + t.print(status) +} + +func (t *jsonPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { + status := summaryOutput{ + MessageType: "summary", + SecondsElapsed: uint64(duration / time.Second), + TotalFiles: filesTotal, + FilesDone: filesFinished, + TotalBytes: allBytesTotal, + BytesDone: allBytesWritten, + } + t.print(status) +} + +type statusUpdate struct { + MessageType string `json:"message_type"` // "status" + SecondsElapsed uint64 `json:"seconds_elapsed,omitempty"` + PercentDone float64 `json:"percent_done"` + TotalFiles uint64 `json:"total_files,omitempty"` + FilesDone uint64 `json:"files_done,omitempty"` + TotalBytes uint64 `json:"total_bytes,omitempty"` + BytesDone uint64 `json:"bytes_done,omitempty"` +} + +type summaryOutput struct { + MessageType string `json:"message_type"` // "summary" + SecondsElapsed uint64 `json:"seconds_elapsed,omitempty"` + TotalFiles uint64 `json:"total_files,omitempty"` + FilesDone uint64 `json:"files_done,omitempty"` + TotalBytes uint64 `json:"total_bytes,omitempty"` + BytesDone uint64 `json:"bytes_done,omitempty"` +} diff --git a/internal/ui/restore/text.go b/internal/ui/restore/text.go index 2f73f7cfe..e6465eed0 100644 --- a/internal/ui/restore/text.go +++ b/internal/ui/restore/text.go @@ -11,7 +11,7 @@ type textPrinter struct { terminal term } -func NewTextPrinter(terminal term) ProgressPrinter { +func NewTextProgress(terminal term) ProgressPrinter { return &textPrinter{ terminal: terminal, } diff --git a/internal/ui/restore/text_test.go b/internal/ui/restore/text_test.go index 0dd32e686..2a8c90878 100644 --- a/internal/ui/restore/text_test.go +++ b/internal/ui/restore/text_test.go @@ -21,21 +21,21 @@ func (m *mockTerm) SetStatus(lines []string) { func TestPrintUpdate(t *testing.T) { term := &mockTerm{} - printer := NewTextPrinter(term) + printer := NewTextProgress(term) printer.Update(3, 11, 29, 47, 5*time.Second) test.Equals(t, []string{"[0:05] 61.70% 3 files 29 B, total 11 files 47 B"}, term.output) } func TestPrintSummaryOnSuccess(t *testing.T) { term := &mockTerm{} - printer := NewTextPrinter(term) + printer := NewTextProgress(term) printer.Finish(11, 11, 47, 47, 5*time.Second) test.Equals(t, []string{"Summary: Restored 11 Files (47 B) in 0:05"}, term.output) } func TestPrintSummaryOnErrors(t *testing.T) { term := &mockTerm{} - printer := NewTextPrinter(term) + printer := NewTextProgress(term) printer.Finish(3, 11, 29, 47, 5*time.Second) test.Equals(t, []string{"Summary: Restored 3 / 11 Files (29 B / 47 B) in 0:05"}, term.output) } From 1531eab7467b7f0e0226293519bffe1d876b1dab Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 1 May 2023 12:19:11 +0200 Subject: [PATCH 19/90] mention restore json support in changelog --- changelog/unreleased/{issue-3627 => issue-426} | 5 +++++ 1 file changed, 5 insertions(+) rename changelog/unreleased/{issue-3627 => issue-426} (66%) diff --git a/changelog/unreleased/issue-3627 b/changelog/unreleased/issue-426 similarity index 66% rename from changelog/unreleased/issue-3627 rename to changelog/unreleased/issue-426 index edbbdbb33..9caf14ef5 100644 --- a/changelog/unreleased/issue-3627 +++ b/changelog/unreleased/issue-426 @@ -4,6 +4,11 @@ The `restore` command now shows a progress report while restoring files. Example: [0:42] 5.76% 23 files 12.98 MiB, total 3456 files 23.54 GiB +JSON output is now also supported. + +https://github.com/restic/restic/issues/426 +https://github.com/restic/restic/issues/3413 https://github.com/restic/restic/issues/3627 https://github.com/restic/restic/pull/3991 +https://github.com/restic/restic/pull/4314 https://forum.restic.net/t/progress-bar-for-restore/5210 From cf162390588263e4ad043d76c09f658fddb9b30c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 13 May 2023 23:23:39 +0200 Subject: [PATCH 20/90] restore: print output via termStatus --- cmd/restic/cmd_restore.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 363b35370..c59ac34de 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -175,6 +175,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, return err } + msg := ui.NewMessage(term, gopts.verbosity) var printer restoreui.ProgressPrinter if gopts.JSON { printer = restoreui.NewJSONProgress(term) @@ -187,7 +188,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, totalErrors := 0 res.Error = func(location string, err error) error { - Warnf("ignoring error for %s: %s\n", location, err) + msg.E("ignoring error for %s: %s\n", location, err) totalErrors++ return nil } @@ -197,12 +198,12 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, selectExcludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { matched, err := filter.List(excludePatterns, item) if err != nil { - Warnf("error for exclude pattern: %v", err) + msg.E("error for exclude pattern: %v", err) } matchedInsensitive, err := filter.List(insensitiveExcludePatterns, strings.ToLower(item)) if err != nil { - Warnf("error for iexclude pattern: %v", err) + msg.E("error for iexclude pattern: %v", err) } // An exclude filter is basically a 'wildcard but foo', @@ -220,12 +221,12 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, selectIncludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { matched, childMayMatch, err := filter.ListWithChild(includePatterns, item) if err != nil { - Warnf("error for include pattern: %v", err) + msg.E("error for include pattern: %v", err) } matchedInsensitive, childMayMatchInsensitive, err := filter.ListWithChild(insensitiveIncludePatterns, strings.ToLower(item)) if err != nil { - Warnf("error for iexclude pattern: %v", err) + msg.E("error for iexclude pattern: %v", err) } selectedForRestore = matched || matchedInsensitive @@ -241,7 +242,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, } if !gopts.JSON { - Verbosef("restoring %s to %s\n", res.Snapshot(), opts.Target) + msg.P("restoring %s to %s\n", res.Snapshot(), opts.Target) } err = res.RestoreTo(ctx, opts.Target) @@ -257,7 +258,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, if opts.Verify { if !gopts.JSON { - Verbosef("verifying files in %s\n", opts.Target) + msg.P("verifying files in %s\n", opts.Target) } var count int t0 := time.Now() @@ -270,7 +271,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, } if !gopts.JSON { - Verbosef("finished verifying %d files in %s (took %s)\n", count, opts.Target, + msg.P("finished verifying %d files in %s (took %s)\n", count, opts.Target, time.Since(t0).Round(time.Millisecond)) } } From b2b0760eb03c3514a938d72a0bf212b29009a474 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 14 May 2023 12:02:34 +0200 Subject: [PATCH 21/90] restore: add test for json output --- internal/ui/restore/json_test.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 internal/ui/restore/json_test.go diff --git a/internal/ui/restore/json_test.go b/internal/ui/restore/json_test.go new file mode 100644 index 000000000..093c87bbb --- /dev/null +++ b/internal/ui/restore/json_test.go @@ -0,0 +1,29 @@ +package restore + +import ( + "testing" + "time" + + "github.com/restic/restic/internal/test" +) + +func TestJSONPrintUpdate(t *testing.T) { + term := &mockTerm{} + printer := NewJSONProgress(term) + printer.Update(3, 11, 29, 47, 5*time.Second) + test.Equals(t, []string{"{\"message_type\":\"status\",\"seconds_elapsed\":5,\"percent_done\":0.6170212765957447,\"total_files\":11,\"files_done\":3,\"total_bytes\":47,\"bytes_done\":29}\n"}, term.output) +} + +func TestJSONPrintSummaryOnSuccess(t *testing.T) { + term := &mockTerm{} + printer := NewJSONProgress(term) + printer.Finish(11, 11, 47, 47, 5*time.Second) + test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_done\":11,\"total_bytes\":47,\"bytes_done\":47}\n"}, term.output) +} + +func TestJSONPrintSummaryOnErrors(t *testing.T) { + term := &mockTerm{} + printer := NewJSONProgress(term) + printer.Finish(3, 11, 29, 47, 5*time.Second) + test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_done\":3,\"total_bytes\":47,\"bytes_done\":29}\n"}, term.output) +} From 07d1f8047e900f48d734595b2f5b84c2d3f46c8c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 9 Jun 2023 12:01:53 +0200 Subject: [PATCH 22/90] restore: More descriptive field names for the JSON output --- internal/ui/restore/json.go | 16 ++++++++-------- internal/ui/restore/json_test.go | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/ui/restore/json.go b/internal/ui/restore/json.go index 0ff0a89cd..c1b95b00b 100644 --- a/internal/ui/restore/json.go +++ b/internal/ui/restore/json.go @@ -25,9 +25,9 @@ func (t *jsonPrinter) Update(filesFinished, filesTotal, allBytesWritten, allByte MessageType: "status", SecondsElapsed: uint64(duration / time.Second), TotalFiles: filesTotal, - FilesDone: filesFinished, + FilesRestored: filesFinished, TotalBytes: allBytesTotal, - BytesDone: allBytesWritten, + BytesRestored: allBytesWritten, } if allBytesTotal > 0 { @@ -42,9 +42,9 @@ func (t *jsonPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allByte MessageType: "summary", SecondsElapsed: uint64(duration / time.Second), TotalFiles: filesTotal, - FilesDone: filesFinished, + FilesRestored: filesFinished, TotalBytes: allBytesTotal, - BytesDone: allBytesWritten, + BytesRestored: allBytesWritten, } t.print(status) } @@ -54,16 +54,16 @@ type statusUpdate struct { SecondsElapsed uint64 `json:"seconds_elapsed,omitempty"` PercentDone float64 `json:"percent_done"` TotalFiles uint64 `json:"total_files,omitempty"` - FilesDone uint64 `json:"files_done,omitempty"` + FilesRestored uint64 `json:"files_restored,omitempty"` TotalBytes uint64 `json:"total_bytes,omitempty"` - BytesDone uint64 `json:"bytes_done,omitempty"` + BytesRestored uint64 `json:"bytes_restored,omitempty"` } type summaryOutput struct { MessageType string `json:"message_type"` // "summary" SecondsElapsed uint64 `json:"seconds_elapsed,omitempty"` TotalFiles uint64 `json:"total_files,omitempty"` - FilesDone uint64 `json:"files_done,omitempty"` + FilesRestored uint64 `json:"files_restored,omitempty"` TotalBytes uint64 `json:"total_bytes,omitempty"` - BytesDone uint64 `json:"bytes_done,omitempty"` + BytesRestored uint64 `json:"bytes_restored,omitempty"` } diff --git a/internal/ui/restore/json_test.go b/internal/ui/restore/json_test.go index 093c87bbb..7bcabb4d7 100644 --- a/internal/ui/restore/json_test.go +++ b/internal/ui/restore/json_test.go @@ -11,19 +11,19 @@ func TestJSONPrintUpdate(t *testing.T) { term := &mockTerm{} printer := NewJSONProgress(term) printer.Update(3, 11, 29, 47, 5*time.Second) - test.Equals(t, []string{"{\"message_type\":\"status\",\"seconds_elapsed\":5,\"percent_done\":0.6170212765957447,\"total_files\":11,\"files_done\":3,\"total_bytes\":47,\"bytes_done\":29}\n"}, term.output) + test.Equals(t, []string{"{\"message_type\":\"status\",\"seconds_elapsed\":5,\"percent_done\":0.6170212765957447,\"total_files\":11,\"files_restored\":3,\"total_bytes\":47,\"bytes_restored\":29}\n"}, term.output) } func TestJSONPrintSummaryOnSuccess(t *testing.T) { term := &mockTerm{} printer := NewJSONProgress(term) printer.Finish(11, 11, 47, 47, 5*time.Second) - test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_done\":11,\"total_bytes\":47,\"bytes_done\":47}\n"}, term.output) + test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":11,\"total_bytes\":47,\"bytes_restored\":47}\n"}, term.output) } func TestJSONPrintSummaryOnErrors(t *testing.T) { term := &mockTerm{} printer := NewJSONProgress(term) printer.Finish(3, 11, 29, 47, 5*time.Second) - test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_done\":3,\"total_bytes\":47,\"bytes_done\":29}\n"}, term.output) + test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":3,\"total_bytes\":47,\"bytes_restored\":29}\n"}, term.output) } From faec0ff816c33602dccf2a8efc64c0cc319b9c63 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 9 Jun 2023 13:17:24 +0200 Subject: [PATCH 23/90] build-release-binaries: support building a subset of all platforms --- helpers/build-release-binaries/main.go | 69 ++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 5 deletions(-) diff --git a/helpers/build-release-binaries/main.go b/helpers/build-release-binaries/main.go index 0c0015f42..94269fe2c 100644 --- a/helpers/build-release-binaries/main.go +++ b/helpers/build-release-binaries/main.go @@ -1,11 +1,14 @@ package main import ( + "errors" "fmt" "os" "os/exec" "path/filepath" "runtime" + "sort" + "strconv" "strings" "time" @@ -14,16 +17,18 @@ import ( ) var opts = struct { - Verbose bool - SourceDir string - OutputDir string - Version string + Verbose bool + SourceDir string + OutputDir string + PlatformSubset string + Version string }{} func init() { pflag.BoolVarP(&opts.Verbose, "verbose", "v", false, "be verbose") pflag.StringVarP(&opts.SourceDir, "source", "s", "/restic", "path to the source code `directory`") pflag.StringVarP(&opts.OutputDir, "output", "o", "/output", "path to the output `directory`") + pflag.StringVar(&opts.PlatformSubset, "platform-subset", "", "specify `n/t` to only build this subset") pflag.StringVar(&opts.Version, "version", "", "use `x.y.z` as the version for output files") pflag.Parse() } @@ -244,15 +249,69 @@ func downloadModules(sourceDir string) { } } +func selectSubset(subset string, target map[string][]string) (map[string][]string, error) { + t, n, _ := strings.Cut(subset, "/") + part, err := strconv.ParseInt(t, 10, 8) + if err != nil { + return nil, fmt.Errorf("failed to parse platform subset %q", subset) + } + total, err := strconv.ParseInt(n, 10, 8) + if err != nil { + return nil, fmt.Errorf("failed to parse platform subset %q", subset) + } + if total < 0 || part < 0 { + return nil, errors.New("platform subset out of range") + } + if part >= total { + return nil, errors.New("t must be in 0 <= t < n") + } + + // flatten platform list + platforms := []string{} + for os, archs := range target { + for _, arch := range archs { + platforms = append(platforms, os+"/"+arch) + } + } + sort.Strings(platforms) + + // select subset + lower := len(platforms) * int(part) / int(total) + upper := len(platforms) * int(part+1) / int(total) + platforms = platforms[lower:upper] + + return buildPlatformList(platforms), nil +} + +func buildPlatformList(platforms []string) map[string][]string { + fmt.Printf("Building for %v\n", platforms) + + targets := make(map[string][]string) + for _, platform := range platforms { + os, arch, _ := strings.Cut(platform, "/") + targets[os] = append(targets[os], arch) + } + return targets +} + func main() { if len(pflag.Args()) != 0 { die("USAGE: build-release-binaries [OPTIONS]") } + targets := defaultBuildTargets + if opts.PlatformSubset != "" { + var err error + targets, err = selectSubset(opts.PlatformSubset, targets) + if err != nil { + die("%s", err) + } + } + sourceDir := abs(opts.SourceDir) outputDir := abs(opts.OutputDir) mkdir(outputDir) downloadModules(sourceDir) - buildTargets(sourceDir, outputDir, defaultBuildTargets) + buildTargets(sourceDir, outputDir, targets) } From 4e9e2c3229b0446f262723cb6c37089f4e479661 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 9 Jun 2023 13:23:49 +0200 Subject: [PATCH 24/90] CI: Use build-release-binaries to run the cross-compilation tests gox silently ignored linux/mips and aix/ppc64. This change also removes the duplicate platform list. --- .github/workflows/tests.yml | 33 +++++++------------------- helpers/build-release-binaries/main.go | 10 ++++++-- 2 files changed, 17 insertions(+), 26 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index aa6ba192a..9bca02f8d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -201,27 +201,19 @@ jobs: cross_compile: strategy: - # ATTENTION: the list of architectures must be in sync with helpers/build-release-binaries/main.go! matrix: # run cross-compile in three batches parallel so the overall tests run faster - targets: - - "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le linux/riscv64 linux/s390x" - - - "openbsd/386 openbsd/amd64 \ - freebsd/386 freebsd/amd64 freebsd/arm \ - aix/ppc64 \ - darwin/amd64 darwin/arm64" - - - "netbsd/386 netbsd/amd64 \ - windows/386 windows/amd64 \ - solaris/amd64" + subset: + - "0/3" + - "1/3" + - "2/3" env: GOPROXY: https://proxy.golang.org runs-on: ubuntu-latest - name: Cross Compile for ${{ matrix.targets }} + name: Cross Compile for subset ${{ matrix.subset }} steps: - name: Set up Go ${{ env.latest_go }} @@ -229,21 +221,14 @@ jobs: with: go-version: ${{ env.latest_go }} - - name: Install gox - run: | - go install github.com/mitchellh/gox@latest - - name: Check out code uses: actions/checkout@v3 - - name: Cross-compile with gox for ${{ matrix.targets }} - env: - GOFLAGS: "-trimpath" - GOX_ARCHS: "${{ matrix.targets }}" + - name: Cross-compile for subset ${{ matrix.subset }} run: | - mkdir build-output - gox -parallel 2 -verbose -osarch "$GOX_ARCHS" -output "build-output/{{.Dir}}_{{.OS}}_{{.Arch}}" ./cmd/restic - gox -parallel 2 -verbose -osarch "$GOX_ARCHS" -tags debug -output "build-output/{{.Dir}}_{{.OS}}_{{.Arch}}_debug" ./cmd/restic + mkdir build-output build-output-debug + go run ./helpers/build-release-binaries/main.go -o build-output -s . --platform-subset ${{ matrix.subset }} + go run ./helpers/build-release-binaries/main.go -o build-output-debug -s . --platform-subset ${{ matrix.subset }} --tags debug lint: name: lint diff --git a/helpers/build-release-binaries/main.go b/helpers/build-release-binaries/main.go index 94269fe2c..914f8e858 100644 --- a/helpers/build-release-binaries/main.go +++ b/helpers/build-release-binaries/main.go @@ -20,6 +20,7 @@ var opts = struct { Verbose bool SourceDir string OutputDir string + Tags string PlatformSubset string Version string }{} @@ -28,6 +29,7 @@ func init() { pflag.BoolVarP(&opts.Verbose, "verbose", "v", false, "be verbose") pflag.StringVarP(&opts.SourceDir, "source", "s", "/restic", "path to the source code `directory`") pflag.StringVarP(&opts.OutputDir, "output", "o", "/output", "path to the output `directory`") + pflag.StringVar(&opts.Tags, "tags", "", "additional build `tags`") pflag.StringVar(&opts.PlatformSubset, "platform-subset", "", "specify `n/t` to only build this subset") pflag.StringVar(&opts.Version, "version", "", "use `x.y.z` as the version for output files") pflag.Parse() @@ -100,10 +102,15 @@ func build(sourceDir, outputDir, goos, goarch string) (filename string) { } outputFile := filepath.Join(outputDir, filename) + tags := "selfupdate" + if opts.Tags != "" { + tags += "," + opts.Tags + } + c := exec.Command("go", "build", "-o", outputFile, "-ldflags", "-s -w", - "-tags", "selfupdate", + "-tags", tags, "./cmd/restic", ) c.Stdout = os.Stdout @@ -225,7 +232,6 @@ func buildTargets(sourceDir, outputDir string, targets map[string][]string) { msg("build finished in %.3fs", time.Since(start).Seconds()) } -// ATTENTION: the list of architectures must be in sync with .github/workflows/tests.yml! var defaultBuildTargets = map[string][]string{ "aix": {"ppc64"}, "darwin": {"amd64", "arm64"}, From 61042a77a4b9f98996bfc4f3742b62a261154aaf Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 9 Jun 2023 13:35:35 +0200 Subject: [PATCH 25/90] building on aix is currently not possible --- helpers/build-release-binaries/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/helpers/build-release-binaries/main.go b/helpers/build-release-binaries/main.go index 914f8e858..6938aff84 100644 --- a/helpers/build-release-binaries/main.go +++ b/helpers/build-release-binaries/main.go @@ -233,7 +233,6 @@ func buildTargets(sourceDir, outputDir string, targets map[string][]string) { } var defaultBuildTargets = map[string][]string{ - "aix": {"ppc64"}, "darwin": {"amd64", "arm64"}, "freebsd": {"386", "amd64", "arm"}, "linux": {"386", "amd64", "arm", "arm64", "ppc64le", "mips", "mipsle", "mips64", "mips64le", "riscv64", "s390x"}, From b2ed42cec45f06b17b86fbab745d85110f2ed3b1 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 16 Jun 2023 23:12:30 +0200 Subject: [PATCH 26/90] index: add basic hat test --- internal/index/indexmap_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/internal/index/indexmap_test.go b/internal/index/indexmap_test.go index 391131ca0..a16670c7d 100644 --- a/internal/index/indexmap_test.go +++ b/internal/index/indexmap_test.go @@ -108,6 +108,21 @@ func TestIndexMapForeachWithID(t *testing.T) { } } +func TestHashedArrayTree(t *testing.T) { + hat := newHAT() + const testSize = 1024 + for i := uint(0); i < testSize; i++ { + rtest.Assert(t, hat.Size() == i, "expected hat size %v got %v", i, hat.Size()) + e, idx := hat.Alloc() + rtest.Assert(t, idx == i, "expected entry at idx %v got %v", i, idx) + e.length = uint32(i) + } + for i := uint(0); i < testSize; i++ { + e := hat.Ref(i) + rtest.Assert(t, e.length == uint32(i), "expected entry to contain %v got %v", uint32(i), e.length) + } +} + func BenchmarkIndexMapHash(b *testing.B) { var m indexMap m.add(restic.ID{}, 0, 0, 0, 0) // Trigger lazy initialization. From 4d5ee987a73c29a747b350c730f007f022ebea00 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 16 Jun 2023 23:19:36 +0200 Subject: [PATCH 27/90] add changelog about missing AIX builds --- changelog/unreleased/pull-4365 | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changelog/unreleased/pull-4365 diff --git a/changelog/unreleased/pull-4365 b/changelog/unreleased/pull-4365 new file mode 100644 index 000000000..c13a80af4 --- /dev/null +++ b/changelog/unreleased/pull-4365 @@ -0,0 +1,6 @@ +Change: Building restic on AIX is temporarily unsupported + +As the current version of the library used for the Azure backend does not +compile on AIX, there are currently no restic builds available for AIX. + +https://github.com/restic/restic/pull/4365 From 56836364a42911a85fba0f78535e60f0a326fd10 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 8 Jun 2023 13:11:34 +0200 Subject: [PATCH 28/90] backend: pass context into every backend constructor --- cmd/restic/global.go | 6 +++--- internal/backend/gs/gs.go | 2 +- internal/backend/gs/gs_test.go | 4 ++-- internal/backend/rclone/backend.go | 12 ++++++------ internal/backend/rclone/backend_test.go | 2 +- internal/backend/rclone/internal_test.go | 4 ++-- internal/backend/rest/rest.go | 4 ++-- internal/backend/rest/rest_int_test.go | 2 +- internal/backend/rest/rest_test.go | 2 +- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 1b9c5b33d..3136e8990 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -584,7 +584,7 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio case "s3": be, err = s3.Open(ctx, *cfg.(*s3.Config), rt) case "gs": - be, err = gs.Open(*cfg.(*gs.Config), rt) + be, err = gs.Open(ctx, *cfg.(*gs.Config), rt) case "azure": be, err = azure.Open(ctx, *cfg.(*azure.Config), rt) case "swift": @@ -592,9 +592,9 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio case "b2": be, err = b2.Open(ctx, *cfg.(*b2.Config), rt) case "rest": - be, err = rest.Open(*cfg.(*rest.Config), rt) + be, err = rest.Open(ctx, *cfg.(*rest.Config), rt) case "rclone": - be, err = rclone.Open(*cfg.(*rclone.Config), lim) + be, err = rclone.Open(ctx, *cfg.(*rclone.Config), lim) default: return nil, errors.Fatalf("invalid backend: %q", loc.Scheme) diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index 7b5489111..022f2534a 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -117,7 +117,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { } // Open opens the gs backend at the specified bucket. -func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Open(_ context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { return open(cfg, rt) } diff --git a/internal/backend/gs/gs_test.go b/internal/backend/gs/gs_test.go index f96b6c62b..a73fe77fb 100644 --- a/internal/backend/gs/gs_test.go +++ b/internal/backend/gs/gs_test.go @@ -58,12 +58,12 @@ func newGSTestSuite(t testing.TB) *test.Suite[gs.Config] { // OpenFn is a function that opens a previously created temporary repository. Open: func(cfg gs.Config) (restic.Backend, error) { - return gs.Open(cfg, tr) + return gs.Open(context.TODO(), cfg, tr) }, // CleanupFn removes data created during the tests. Cleanup: func(cfg gs.Config) error { - be, err := gs.Open(cfg, tr) + be, err := gs.Open(context.TODO(), cfg, tr) if err != nil { return err } diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go index 085c89945..881990bc3 100644 --- a/internal/backend/rclone/backend.go +++ b/internal/backend/rclone/backend.go @@ -134,7 +134,7 @@ func wrapConn(c *StdioConn, lim limiter.Limiter) *wrappedConn { } // New initializes a Backend and starts the process. -func newBackend(cfg Config, lim limiter.Limiter) (*Backend, error) { +func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter) (*Backend, error) { var ( args []string err error @@ -197,7 +197,7 @@ func newBackend(cfg Config, lim limiter.Limiter) (*Backend, error) { wg: wg, } - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(ctx) defer cancel() wg.Add(1) @@ -256,8 +256,8 @@ func newBackend(cfg Config, lim limiter.Limiter) (*Backend, error) { } // Open starts an rclone process with the given config. -func Open(cfg Config, lim limiter.Limiter) (*Backend, error) { - be, err := newBackend(cfg, lim) +func Open(ctx context.Context, cfg Config, lim limiter.Limiter) (*Backend, error) { + be, err := newBackend(ctx, cfg, lim) if err != nil { return nil, err } @@ -272,7 +272,7 @@ func Open(cfg Config, lim limiter.Limiter) (*Backend, error) { URL: url, } - restBackend, err := rest.Open(restConfig, debug.RoundTripper(be.tr)) + restBackend, err := rest.Open(ctx, restConfig, debug.RoundTripper(be.tr)) if err != nil { _ = be.Close() return nil, err @@ -284,7 +284,7 @@ func Open(cfg Config, lim limiter.Limiter) (*Backend, error) { // Create initializes a new restic repo with rclone. func Create(ctx context.Context, cfg Config) (*Backend, error) { - be, err := newBackend(cfg, nil) + be, err := newBackend(ctx, cfg, nil) if err != nil { return nil, err } diff --git a/internal/backend/rclone/backend_test.go b/internal/backend/rclone/backend_test.go index c497271f6..a562a99e6 100644 --- a/internal/backend/rclone/backend_test.go +++ b/internal/backend/rclone/backend_test.go @@ -39,7 +39,7 @@ func newTestSuite(t testing.TB) *test.Suite[rclone.Config] { // OpenFn is a function that opens a previously created temporary repository. Open: func(cfg rclone.Config) (restic.Backend, error) { t.Logf("Open()") - return rclone.Open(cfg, nil) + return rclone.Open(context.TODO(), cfg, nil) }, } } diff --git a/internal/backend/rclone/internal_test.go b/internal/backend/rclone/internal_test.go index bfec2b98c..32fe850a0 100644 --- a/internal/backend/rclone/internal_test.go +++ b/internal/backend/rclone/internal_test.go @@ -15,7 +15,7 @@ func TestRcloneExit(t *testing.T) { dir := rtest.TempDir(t) cfg := NewConfig() cfg.Remote = dir - be, err := Open(cfg, nil) + be, err := Open(context.TODO(), cfg, nil) var e *exec.Error if errors.As(err, &e) && e.Err == exec.ErrNotFound { t.Skipf("program %q not found", e.Name) @@ -45,7 +45,7 @@ func TestRcloneFailedStart(t *testing.T) { cfg := NewConfig() // exits with exit code 1 cfg.Program = "false" - _, err := Open(cfg, nil) + _, err := Open(context.TODO(), cfg, nil) var e *exec.ExitError if !errors.As(err, &e) { // unexpected error diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index 68397cd1b..0906a7f10 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -36,7 +36,7 @@ const ( ) // Open opens the REST backend with the given config. -func Open(cfg Config, rt http.RoundTripper) (*Backend, error) { +func Open(_ context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) { // use url without trailing slash for layout url := cfg.URL.String() if url[len(url)-1] == '/' { @@ -55,7 +55,7 @@ func Open(cfg Config, rt http.RoundTripper) (*Backend, error) { // Create creates a new REST on server configured in config. func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) { - be, err := Open(cfg, rt) + be, err := Open(ctx, cfg, rt) if err != nil { return nil, err } diff --git a/internal/backend/rest/rest_int_test.go b/internal/backend/rest/rest_int_test.go index 7184f5fbe..e7810c5e3 100644 --- a/internal/backend/rest/rest_int_test.go +++ b/internal/backend/rest/rest_int_test.go @@ -117,7 +117,7 @@ func TestListAPI(t *testing.T) { URL: srvURL, } - be, err := rest.Open(cfg, http.DefaultTransport) + be, err := rest.Open(context.TODO(), cfg, http.DefaultTransport) if err != nil { t.Fatal(err) } diff --git a/internal/backend/rest/rest_test.go b/internal/backend/rest/rest_test.go index 2ebd00f5e..4d069e63c 100644 --- a/internal/backend/rest/rest_test.go +++ b/internal/backend/rest/rest_test.go @@ -90,7 +90,7 @@ func newTestSuite(_ context.Context, t testing.TB, url *url.URL, minimalData boo // OpenFn is a function that opens a previously created temporary repository. Open: func(cfg rest.Config) (restic.Backend, error) { - return rest.Open(cfg, tr) + return rest.Open(context.TODO(), cfg, tr) }, // CleanupFn removes data created during the tests. From 7d12c292863efbd51c28781ce8be478bb404b994 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 8 Jun 2023 13:04:34 +0200 Subject: [PATCH 29/90] backend: Unify backend construction using factory and registry This unified construction removes most backend-specific code from global.go. The backend registry will also enable integration tests to use custom backends if necessary. --- cmd/restic/cmd_init.go | 10 +-- cmd/restic/global.go | 76 ++++++----------- cmd/restic/integration_helpers_test.go | 2 + internal/backend/azure/azure.go | 5 ++ internal/backend/b2/b2.go | 5 ++ internal/backend/gs/gs.go | 5 ++ internal/backend/local/local.go | 10 +++ internal/backend/location/location.go | 67 ++++----------- internal/backend/location/location_test.go | 67 +++++++-------- internal/backend/location/registry.go | 94 ++++++++++++++++++++++ internal/backend/rclone/backend.go | 9 ++- internal/backend/rclone/backend_test.go | 2 +- internal/backend/rest/rest.go | 5 ++ internal/backend/s3/s3.go | 5 ++ internal/backend/sftp/sftp.go | 10 +++ internal/backend/swift/swift.go | 5 ++ 16 files changed, 235 insertions(+), 142 deletions(-) create mode 100644 internal/backend/location/registry.go diff --git a/cmd/restic/cmd_init.go b/cmd/restic/cmd_init.go index 43de7ff89..b9dabdc2d 100644 --- a/cmd/restic/cmd_init.go +++ b/cmd/restic/cmd_init.go @@ -87,9 +87,9 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args [] return err } - be, err := create(ctx, repo, gopts.extended) + be, err := create(ctx, repo, gopts, gopts.extended) if err != nil { - return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err) + return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.backends, gopts.Repo), err) } s, err := repository.New(be, repository.Options{ @@ -102,11 +102,11 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args [] err = s.Init(ctx, version, gopts.password, chunkerPolynomial) if err != nil { - return errors.Fatalf("create key in repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err) + return errors.Fatalf("create key in repository at %s failed: %v\n", location.StripPassword(gopts.backends, gopts.Repo), err) } if !gopts.JSON { - Verbosef("created restic repository %v at %s", s.Config().ID[:10], location.StripPassword(gopts.Repo)) + Verbosef("created restic repository %v at %s", s.Config().ID[:10], location.StripPassword(gopts.backends, gopts.Repo)) if opts.CopyChunkerParameters && chunkerPolynomial != nil { Verbosef(" with chunker parameters copied from secondary repository\n") } else { @@ -121,7 +121,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args [] status := initSuccess{ MessageType: "initialized", ID: s.Config().ID, - Repository: location.StripPassword(gopts.Repo), + Repository: location.StripPassword(gopts.backends, gopts.Repo), } return json.NewEncoder(globalOptions.stdout).Encode(status) } diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 3136e8990..f4886ec90 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -75,6 +75,7 @@ type GlobalOptions struct { stdout io.Writer stderr io.Writer + backends *location.Registry backendTestHook, backendInnerTestHook backendWrapper // verbosity is set as follows: @@ -98,6 +99,18 @@ var isReadingPassword bool var internalGlobalCtx context.Context func init() { + backends := location.NewRegistry() + backends.Register("b2", b2.NewFactory()) + backends.Register("local", local.NewFactory()) + backends.Register("sftp", sftp.NewFactory()) + backends.Register("s3", s3.NewFactory()) + backends.Register("gs", gs.NewFactory()) + backends.Register("azure", azure.NewFactory()) + backends.Register("swift", swift.NewFactory()) + backends.Register("rest", rest.NewFactory()) + backends.Register("rclone", rclone.NewFactory()) + globalOptions.backends = backends + var cancel context.CancelFunc internalGlobalCtx, cancel = context.WithCancel(context.Background()) AddCleanupHandler(func(code int) (int, error) { @@ -554,8 +567,8 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro // Open the backend specified by a location config. func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) { - debug.Log("parsing location %v", location.StripPassword(s)) - loc, err := location.Parse(s) + debug.Log("parsing location %v", location.StripPassword(gopts.backends, s)) + loc, err := location.Parse(gopts.backends, s) if err != nil { return nil, errors.Fatalf("parsing repository location failed: %v", err) } @@ -576,32 +589,14 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio lim := limiter.NewStaticLimiter(gopts.Limits) rt = lim.Transport(rt) - switch loc.Scheme { - case "local": - be, err = local.Open(ctx, *cfg.(*local.Config)) - case "sftp": - be, err = sftp.Open(ctx, *cfg.(*sftp.Config)) - case "s3": - be, err = s3.Open(ctx, *cfg.(*s3.Config), rt) - case "gs": - be, err = gs.Open(ctx, *cfg.(*gs.Config), rt) - case "azure": - be, err = azure.Open(ctx, *cfg.(*azure.Config), rt) - case "swift": - be, err = swift.Open(ctx, *cfg.(*swift.Config), rt) - case "b2": - be, err = b2.Open(ctx, *cfg.(*b2.Config), rt) - case "rest": - be, err = rest.Open(ctx, *cfg.(*rest.Config), rt) - case "rclone": - be, err = rclone.Open(ctx, *cfg.(*rclone.Config), lim) - - default: + factory := gopts.backends.Lookup(loc.Scheme) + if factory == nil { return nil, errors.Fatalf("invalid backend: %q", loc.Scheme) } + be, err = factory.Open(ctx, cfg, rt, lim) if err != nil { - return nil, errors.Fatalf("unable to open repository at %v: %v", location.StripPassword(s), err) + return nil, errors.Fatalf("unable to open repository at %v: %v", location.StripPassword(gopts.backends, s), err) } // wrap with debug logging and connection limiting @@ -623,7 +618,7 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio // check if config is there fi, err := be.Stat(ctx, restic.Handle{Type: restic.ConfigFile}) if err != nil { - return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(s)) + return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(gopts.backends, s)) } if fi.Size == 0 { @@ -634,9 +629,9 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio } // Create the backend specified by URI. -func create(ctx context.Context, s string, opts options.Options) (restic.Backend, error) { +func create(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) { debug.Log("parsing location %v", s) - loc, err := location.Parse(s) + loc, err := location.Parse(gopts.backends, s) if err != nil { return nil, err } @@ -651,31 +646,12 @@ func create(ctx context.Context, s string, opts options.Options) (restic.Backend return nil, err } - var be restic.Backend - switch loc.Scheme { - case "local": - be, err = local.Create(ctx, *cfg.(*local.Config)) - case "sftp": - be, err = sftp.Create(ctx, *cfg.(*sftp.Config)) - case "s3": - be, err = s3.Create(ctx, *cfg.(*s3.Config), rt) - case "gs": - be, err = gs.Create(ctx, *cfg.(*gs.Config), rt) - case "azure": - be, err = azure.Create(ctx, *cfg.(*azure.Config), rt) - case "swift": - be, err = swift.Open(ctx, *cfg.(*swift.Config), rt) - case "b2": - be, err = b2.Create(ctx, *cfg.(*b2.Config), rt) - case "rest": - be, err = rest.Create(ctx, *cfg.(*rest.Config), rt) - case "rclone": - be, err = rclone.Create(ctx, *cfg.(*rclone.Config)) - default: - debug.Log("invalid repository scheme: %v", s) - return nil, errors.Fatalf("invalid scheme %q", loc.Scheme) + factory := gopts.backends.Lookup(loc.Scheme) + if factory == nil { + return nil, errors.Fatalf("invalid backend: %q", loc.Scheme) } + be, err := factory.Create(ctx, cfg, rt, nil) if err != nil { return nil, err } diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go index a0e4d49d6..b7cb5b333 100644 --- a/cmd/restic/integration_helpers_test.go +++ b/cmd/restic/integration_helpers_test.go @@ -206,6 +206,8 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) { // replace this hook with "nil" if listing a filetype more than once is necessary backendTestHook: func(r restic.Backend) (restic.Backend, error) { return newOrderedListOnceBackend(r), nil }, + // start with default set of backends + backends: globalOptions.backends, } // always overwrite global options diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index 4041d3adc..b33b8dca6 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -14,6 +14,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -43,6 +44,10 @@ const defaultListMaxItems = 5000 // make sure that *Backend implements backend.Backend var _ restic.Backend = &Backend{} +func NewFactory() location.Factory { + return location.NewHTTPBackendFactory(ParseConfig, location.NoPassword, Create, Open) +} + func open(cfg Config, rt http.RoundTripper) (*Backend, error) { debug.Log("open, config %#v", cfg) var client *azContainer.Client diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go index 7f4dba831..eb2cfe3c2 100644 --- a/internal/backend/b2/b2.go +++ b/internal/backend/b2/b2.go @@ -11,6 +11,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -36,6 +37,10 @@ const defaultListMaxItems = 10 * 1000 // ensure statically that *b2Backend implements restic.Backend. var _ restic.Backend = &b2Backend{} +func NewFactory() location.Factory { + return location.NewHTTPBackendFactory(ParseConfig, location.NoPassword, Create, Open) +} + type sniffingRoundTripper struct { sync.Mutex lastErr error diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index 022f2534a..445ccc77d 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -15,6 +15,7 @@ import ( "github.com/pkg/errors" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" @@ -47,6 +48,10 @@ type Backend struct { // Ensure that *Backend implements restic.Backend. var _ restic.Backend = &Backend{} +func NewFactory() location.Factory { + return location.NewHTTPBackendFactory(ParseConfig, location.NoPassword, Create, Open) +} + func getStorageClient(rt http.RoundTripper) (*storage.Client, error) { // create a new HTTP client httpClient := &http.Client{ diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index d6bdef1e4..02ac81b8d 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -10,6 +10,8 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" + "github.com/restic/restic/internal/backend/limiter" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" @@ -28,6 +30,14 @@ type Local struct { // ensure statically that *Local implements restic.Backend. var _ restic.Backend = &Local{} +func NewFactory() location.Factory { + return location.NewLimitedBackendFactory(ParseConfig, location.NoPassword, func(ctx context.Context, cfg Config, _ limiter.Limiter) (*Local, error) { + return Create(ctx, cfg) + }, func(ctx context.Context, cfg Config, _ limiter.Limiter) (*Local, error) { + return Open(ctx, cfg) + }) +} + const defaultLayout = "default" func open(ctx context.Context, cfg Config) (*Local, error) { diff --git a/internal/backend/location/location.go b/internal/backend/location/location.go index 612ae1b4c..947ca17c3 100644 --- a/internal/backend/location/location.go +++ b/internal/backend/location/location.go @@ -4,15 +4,6 @@ package location import ( "strings" - "github.com/restic/restic/internal/backend/azure" - "github.com/restic/restic/internal/backend/b2" - "github.com/restic/restic/internal/backend/gs" - "github.com/restic/restic/internal/backend/local" - "github.com/restic/restic/internal/backend/rclone" - "github.com/restic/restic/internal/backend/rest" - "github.com/restic/restic/internal/backend/s3" - "github.com/restic/restic/internal/backend/sftp" - "github.com/restic/restic/internal/backend/swift" "github.com/restic/restic/internal/errors" ) @@ -23,34 +14,8 @@ type Location struct { Config interface{} } -type parser struct { - scheme string - parse func(string) (interface{}, error) - stripPassword func(string) string -} - -func configToAny[C any](parser func(string) (*C, error)) func(string) (interface{}, error) { - return func(s string) (interface{}, error) { - return parser(s) - } -} - -// parsers is a list of valid config parsers for the backends. The first parser -// is the fallback and should always be set to the local backend. -var parsers = []parser{ - {"b2", configToAny(b2.ParseConfig), noPassword}, - {"local", configToAny(local.ParseConfig), noPassword}, - {"sftp", configToAny(sftp.ParseConfig), noPassword}, - {"s3", configToAny(s3.ParseConfig), noPassword}, - {"gs", configToAny(gs.ParseConfig), noPassword}, - {"azure", configToAny(azure.ParseConfig), noPassword}, - {"swift", configToAny(swift.ParseConfig), noPassword}, - {"rest", configToAny(rest.ParseConfig), rest.StripPassword}, - {"rclone", configToAny(rclone.ParseConfig), noPassword}, -} - -// noPassword returns the repository location unchanged (there's no sensitive information there) -func noPassword(s string) string { +// NoPassword returns the repository location unchanged (there's no sensitive information there) +func NoPassword(s string) string { return s } @@ -88,16 +53,13 @@ func isPath(s string) bool { // starts with a backend name followed by a colon, that backend's Parse() // function is called. Otherwise, the local backend is used which interprets s // as the name of a directory. -func Parse(s string) (u Location, err error) { +func Parse(registry *Registry, s string) (u Location, err error) { scheme := extractScheme(s) u.Scheme = scheme - for _, parser := range parsers { - if parser.scheme != scheme { - continue - } - - u.Config, err = parser.parse(s) + factory := registry.Lookup(scheme) + if factory != nil { + u.Config, err = factory.ParseConfig(s) if err != nil { return Location{}, err } @@ -111,7 +73,12 @@ func Parse(s string) (u Location, err error) { } u.Scheme = "local" - u.Config, err = local.ParseConfig("local:" + s) + factory = registry.Lookup(u.Scheme) + if factory == nil { + return Location{}, errors.New("local backend not available") + } + + u.Config, err = factory.ParseConfig("local:" + s) if err != nil { return Location{}, err } @@ -120,14 +87,12 @@ func Parse(s string) (u Location, err error) { } // StripPassword returns a displayable version of a repository location (with any sensitive information removed) -func StripPassword(s string) string { +func StripPassword(registry *Registry, s string) string { scheme := extractScheme(s) - for _, parser := range parsers { - if parser.scheme != scheme { - continue - } - return parser.stripPassword(s) + factory := registry.Lookup(scheme) + if factory != nil { + return factory.StripPassword(s) } return s } diff --git a/internal/backend/location/location_test.go b/internal/backend/location/location_test.go index 9f5db70c9..6e9042200 100644 --- a/internal/backend/location/location_test.go +++ b/internal/backend/location/location_test.go @@ -1,4 +1,4 @@ -package location +package location_test import ( "net/url" @@ -7,6 +7,7 @@ import ( "github.com/restic/restic/internal/backend/b2" "github.com/restic/restic/internal/backend/local" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/rest" "github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/backend/sftp" @@ -24,11 +25,11 @@ func parseURL(s string) *url.URL { var parseTests = []struct { s string - u Location + u location.Location }{ { "local:/srv/repo", - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: "/srv/repo", Connections: 2, @@ -37,7 +38,7 @@ var parseTests = []struct { }, { "local:dir1/dir2", - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: "dir1/dir2", Connections: 2, @@ -46,7 +47,7 @@ var parseTests = []struct { }, { "local:dir1/dir2", - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: "dir1/dir2", Connections: 2, @@ -55,7 +56,7 @@ var parseTests = []struct { }, { "dir1/dir2", - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: "dir1/dir2", Connections: 2, @@ -64,7 +65,7 @@ var parseTests = []struct { }, { "/dir1/dir2", - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: "/dir1/dir2", Connections: 2, @@ -73,7 +74,7 @@ var parseTests = []struct { }, { "local:../dir1/dir2", - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: "../dir1/dir2", Connections: 2, @@ -82,7 +83,7 @@ var parseTests = []struct { }, { "/dir1/dir2", - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: "/dir1/dir2", Connections: 2, @@ -91,7 +92,7 @@ var parseTests = []struct { }, { "/dir1:foobar/dir2", - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: "/dir1:foobar/dir2", Connections: 2, @@ -100,7 +101,7 @@ var parseTests = []struct { }, { `\dir1\foobar\dir2`, - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: `\dir1\foobar\dir2`, Connections: 2, @@ -109,7 +110,7 @@ var parseTests = []struct { }, { `c:\dir1\foobar\dir2`, - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: `c:\dir1\foobar\dir2`, Connections: 2, @@ -118,7 +119,7 @@ var parseTests = []struct { }, { `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`, - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`, Connections: 2, @@ -127,7 +128,7 @@ var parseTests = []struct { }, { `c:/dir1/foobar/dir2`, - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: `c:/dir1/foobar/dir2`, Connections: 2, @@ -136,7 +137,7 @@ var parseTests = []struct { }, { "sftp:user@host:/srv/repo", - Location{Scheme: "sftp", + location.Location{Scheme: "sftp", Config: &sftp.Config{ User: "user", Host: "host", @@ -147,7 +148,7 @@ var parseTests = []struct { }, { "sftp:host:/srv/repo", - Location{Scheme: "sftp", + location.Location{Scheme: "sftp", Config: &sftp.Config{ User: "", Host: "host", @@ -158,7 +159,7 @@ var parseTests = []struct { }, { "sftp://user@host/srv/repo", - Location{Scheme: "sftp", + location.Location{Scheme: "sftp", Config: &sftp.Config{ User: "user", Host: "host", @@ -169,7 +170,7 @@ var parseTests = []struct { }, { "sftp://user@host//srv/repo", - Location{Scheme: "sftp", + location.Location{Scheme: "sftp", Config: &sftp.Config{ User: "user", Host: "host", @@ -181,7 +182,7 @@ var parseTests = []struct { { "s3://eu-central-1/bucketname", - Location{Scheme: "s3", + location.Location{Scheme: "s3", Config: &s3.Config{ Endpoint: "eu-central-1", Bucket: "bucketname", @@ -192,7 +193,7 @@ var parseTests = []struct { }, { "s3://hostname.foo/bucketname", - Location{Scheme: "s3", + location.Location{Scheme: "s3", Config: &s3.Config{ Endpoint: "hostname.foo", Bucket: "bucketname", @@ -203,7 +204,7 @@ var parseTests = []struct { }, { "s3://hostname.foo/bucketname/prefix/directory", - Location{Scheme: "s3", + location.Location{Scheme: "s3", Config: &s3.Config{ Endpoint: "hostname.foo", Bucket: "bucketname", @@ -214,7 +215,7 @@ var parseTests = []struct { }, { "s3:eu-central-1/repo", - Location{Scheme: "s3", + location.Location{Scheme: "s3", Config: &s3.Config{ Endpoint: "eu-central-1", Bucket: "repo", @@ -225,7 +226,7 @@ var parseTests = []struct { }, { "s3:eu-central-1/repo/prefix/directory", - Location{Scheme: "s3", + location.Location{Scheme: "s3", Config: &s3.Config{ Endpoint: "eu-central-1", Bucket: "repo", @@ -236,7 +237,7 @@ var parseTests = []struct { }, { "s3:https://hostname.foo/repo", - Location{Scheme: "s3", + location.Location{Scheme: "s3", Config: &s3.Config{ Endpoint: "hostname.foo", Bucket: "repo", @@ -247,7 +248,7 @@ var parseTests = []struct { }, { "s3:https://hostname.foo/repo/prefix/directory", - Location{Scheme: "s3", + location.Location{Scheme: "s3", Config: &s3.Config{ Endpoint: "hostname.foo", Bucket: "repo", @@ -258,7 +259,7 @@ var parseTests = []struct { }, { "s3:http://hostname.foo/repo", - Location{Scheme: "s3", + location.Location{Scheme: "s3", Config: &s3.Config{ Endpoint: "hostname.foo", Bucket: "repo", @@ -270,7 +271,7 @@ var parseTests = []struct { }, { "swift:container17:/", - Location{Scheme: "swift", + location.Location{Scheme: "swift", Config: &swift.Config{ Container: "container17", Prefix: "", @@ -280,7 +281,7 @@ var parseTests = []struct { }, { "swift:container17:/prefix97", - Location{Scheme: "swift", + location.Location{Scheme: "swift", Config: &swift.Config{ Container: "container17", Prefix: "prefix97", @@ -290,7 +291,7 @@ var parseTests = []struct { }, { "rest:http://hostname.foo:1234/", - Location{Scheme: "rest", + location.Location{Scheme: "rest", Config: &rest.Config{ URL: parseURL("http://hostname.foo:1234/"), Connections: 5, @@ -298,7 +299,7 @@ var parseTests = []struct { }, }, { - "b2:bucketname:/prefix", Location{Scheme: "b2", + "b2:bucketname:/prefix", location.Location{Scheme: "b2", Config: &b2.Config{ Bucket: "bucketname", Prefix: "prefix", @@ -307,7 +308,7 @@ var parseTests = []struct { }, }, { - "b2:bucketname", Location{Scheme: "b2", + "b2:bucketname", location.Location{Scheme: "b2", Config: &b2.Config{ Bucket: "bucketname", Prefix: "", @@ -320,7 +321,7 @@ var parseTests = []struct { func TestParse(t *testing.T) { for i, test := range parseTests { t.Run(test.s, func(t *testing.T) { - u, err := Parse(test.s) + u, err := location.Parse(test.s) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -346,7 +347,7 @@ func TestInvalidScheme(t *testing.T) { for _, s := range invalidSchemes { t.Run(s, func(t *testing.T) { - _, err := Parse(s) + _, err := location.Parse(s) if err == nil { t.Fatalf("error for invalid location %q not found", s) } diff --git a/internal/backend/location/registry.go b/internal/backend/location/registry.go new file mode 100644 index 000000000..5d644dfb9 --- /dev/null +++ b/internal/backend/location/registry.go @@ -0,0 +1,94 @@ +package location + +import ( + "context" + "net/http" + + "github.com/restic/restic/internal/backend/limiter" + "github.com/restic/restic/internal/restic" +) + +type Registry struct { + factories map[string]Factory +} + +func NewRegistry() *Registry { + return &Registry{ + factories: make(map[string]Factory), + } +} + +func (r *Registry) Register(scheme string, factory Factory) { + if r.factories[scheme] != nil { + panic("duplicate backend") + } + r.factories[scheme] = factory +} + +func (r *Registry) Lookup(scheme string) Factory { + return r.factories[scheme] +} + +type Factory interface { + ParseConfig(s string) (interface{}, error) + StripPassword(s string) string + Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) + Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) +} + +type GenericBackendFactory[C any, T restic.Backend] struct { + parseConfigFn func(s string) (*C, error) + stripPasswordFn func(s string) string + createFn func(ctx context.Context, cfg C, rt http.RoundTripper, lim limiter.Limiter) (T, error) + openFn func(ctx context.Context, cfg C, rt http.RoundTripper, lim limiter.Limiter) (T, error) +} + +func (f *GenericBackendFactory[C, T]) ParseConfig(s string) (interface{}, error) { + return f.parseConfigFn(s) +} +func (f *GenericBackendFactory[C, T]) StripPassword(s string) string { + if f.stripPasswordFn != nil { + return f.stripPasswordFn(s) + } + return s +} +func (f *GenericBackendFactory[C, T]) Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) { + return f.createFn(ctx, *cfg.(*C), rt, lim) +} +func (f *GenericBackendFactory[C, T]) Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) { + return f.openFn(ctx, *cfg.(*C), rt, lim) +} + +func NewHTTPBackendFactory[C any, T restic.Backend](parseConfigFn func(s string) (*C, error), + stripPasswordFn func(s string) string, + createFn func(ctx context.Context, cfg C, rt http.RoundTripper) (T, error), + openFn func(ctx context.Context, cfg C, rt http.RoundTripper) (T, error)) *GenericBackendFactory[C, T] { + + return &GenericBackendFactory[C, T]{ + parseConfigFn: parseConfigFn, + stripPasswordFn: stripPasswordFn, + createFn: func(ctx context.Context, cfg C, rt http.RoundTripper, _ limiter.Limiter) (T, error) { + return createFn(ctx, cfg, rt) + }, + openFn: func(ctx context.Context, cfg C, rt http.RoundTripper, _ limiter.Limiter) (T, error) { + return openFn(ctx, cfg, rt) + }, + } +} + +func NewLimitedBackendFactory[C any, T restic.Backend](parseConfigFn func(s string) (*C, error), + stripPasswordFn func(s string) string, + createFn func(ctx context.Context, cfg C, lim limiter.Limiter) (T, error), + openFn func(ctx context.Context, cfg C, lim limiter.Limiter) (T, error)) *GenericBackendFactory[C, T] { + + return &GenericBackendFactory[C, T]{ + parseConfigFn: parseConfigFn, + stripPasswordFn: stripPasswordFn, + createFn: func(ctx context.Context, cfg C, _ http.RoundTripper, lim limiter.Limiter) (T, error) { + return createFn(ctx, cfg, lim) + }, + openFn: func(ctx context.Context, cfg C, _ http.RoundTripper, lim limiter.Limiter) (T, error) { + return openFn(ctx, cfg, lim) + }, + } +} diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go index 881990bc3..f3a97ef75 100644 --- a/internal/backend/rclone/backend.go +++ b/internal/backend/rclone/backend.go @@ -19,6 +19,7 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/limiter" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/rest" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" @@ -36,6 +37,10 @@ type Backend struct { conn *StdioConn } +func NewFactory() location.Factory { + return location.NewLimitedBackendFactory(ParseConfig, location.NoPassword, Create, Open) +} + // run starts command with args and initializes the StdioConn. func run(command string, args ...string) (*StdioConn, *sync.WaitGroup, chan struct{}, func() error, error) { cmd := exec.Command(command, args...) @@ -283,8 +288,8 @@ func Open(ctx context.Context, cfg Config, lim limiter.Limiter) (*Backend, error } // Create initializes a new restic repo with rclone. -func Create(ctx context.Context, cfg Config) (*Backend, error) { - be, err := newBackend(ctx, cfg, nil) +func Create(ctx context.Context, cfg Config, lim limiter.Limiter) (*Backend, error) { + be, err := newBackend(ctx, cfg, lim) if err != nil { return nil, err } diff --git a/internal/backend/rclone/backend_test.go b/internal/backend/rclone/backend_test.go index a562a99e6..738462577 100644 --- a/internal/backend/rclone/backend_test.go +++ b/internal/backend/rclone/backend_test.go @@ -27,7 +27,7 @@ func newTestSuite(t testing.TB) *test.Suite[rclone.Config] { // CreateFn is a function that creates a temporary repository for the tests. Create: func(cfg rclone.Config) (restic.Backend, error) { t.Logf("Create()") - be, err := rclone.Create(context.TODO(), cfg) + be, err := rclone.Create(context.TODO(), cfg, nil) var e *exec.Error if errors.As(err, &e) && e.Err == exec.ErrNotFound { t.Skipf("program %q not found", e.Name) diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index 0906a7f10..4fb2d54de 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -13,6 +13,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -29,6 +30,10 @@ type Backend struct { layout.Layout } +func NewFactory() location.Factory { + return location.NewHTTPBackendFactory(ParseConfig, StripPassword, Create, Open) +} + // the REST API protocol version is decided by HTTP request headers, these are the constants. const ( ContentTypeV1 = "application/vnd.x.restic.rest.v1" diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index 7b7a761ce..dd5cc36e6 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -13,6 +13,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -31,6 +32,10 @@ type Backend struct { // make sure that *Backend implements backend.Backend var _ restic.Backend = &Backend{} +func NewFactory() location.Factory { + return location.NewHTTPBackendFactory(ParseConfig, location.NoPassword, Create, Open) +} + const defaultLayout = "default" func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) { diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index 12c355003..f0a7ef9bc 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -15,6 +15,8 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" + "github.com/restic/restic/internal/backend/limiter" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -41,6 +43,14 @@ type SFTP struct { var _ restic.Backend = &SFTP{} +func NewFactory() location.Factory { + return location.NewLimitedBackendFactory(ParseConfig, location.NoPassword, func(ctx context.Context, cfg Config, _ limiter.Limiter) (*SFTP, error) { + return Create(ctx, cfg) + }, func(ctx context.Context, cfg Config, _ limiter.Limiter) (*SFTP, error) { + return Open(ctx, cfg) + }) +} + const defaultLayout = "default" func startClient(cfg Config) (*SFTP, error) { diff --git a/internal/backend/swift/swift.go b/internal/backend/swift/swift.go index cfa9ed665..019456be7 100644 --- a/internal/backend/swift/swift.go +++ b/internal/backend/swift/swift.go @@ -15,6 +15,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -34,6 +35,10 @@ type beSwift struct { // ensure statically that *beSwift implements restic.Backend. var _ restic.Backend = &beSwift{} +func NewFactory() location.Factory { + return location.NewHTTPBackendFactory(ParseConfig, location.NoPassword, Open, Open) +} + // Open opens the swift backend at a container in region. The container is // created if it does not exist yet. func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { From 9aa9e0d1ec6c8453406337756f211eaaeaffcf83 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 8 Jun 2023 13:06:25 +0200 Subject: [PATCH 30/90] local/sftp: move limiter setup into backend --- cmd/restic/global.go | 5 ----- internal/backend/limiter/limiter_backend.go | 15 +++++++++++++++ internal/backend/local/local.go | 6 +----- internal/backend/sftp/sftp.go | 6 +----- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index f4886ec90..f08a75b13 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -610,11 +610,6 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio } } - if loc.Scheme == "local" || loc.Scheme == "sftp" { - // wrap the backend in a LimitBackend so that the throughput is limited - be = limiter.LimitBackend(be, lim) - } - // check if config is there fi, err := be.Stat(ctx, restic.Handle{Type: restic.ConfigFile}) if err != nil { diff --git a/internal/backend/limiter/limiter_backend.go b/internal/backend/limiter/limiter_backend.go index 7fcca59cc..a91794037 100644 --- a/internal/backend/limiter/limiter_backend.go +++ b/internal/backend/limiter/limiter_backend.go @@ -7,6 +7,21 @@ import ( "github.com/restic/restic/internal/restic" ) +func WrapBackendConstructor[B restic.Backend, C any](constructor func(ctx context.Context, cfg C) (B, error)) func(ctx context.Context, cfg C, lim Limiter) (restic.Backend, error) { + return func(ctx context.Context, cfg C, lim Limiter) (restic.Backend, error) { + var be restic.Backend + be, err := constructor(ctx, cfg) + if err != nil { + return nil, err + } + + if lim != nil { + be = LimitBackend(be, lim) + } + return be, nil + } +} + // LimitBackend wraps a Backend and applies rate limiting to Load() and Save() // calls on the backend. func LimitBackend(be restic.Backend, l Limiter) restic.Backend { diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index 02ac81b8d..e9d00abf7 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -31,11 +31,7 @@ type Local struct { var _ restic.Backend = &Local{} func NewFactory() location.Factory { - return location.NewLimitedBackendFactory(ParseConfig, location.NoPassword, func(ctx context.Context, cfg Config, _ limiter.Limiter) (*Local, error) { - return Create(ctx, cfg) - }, func(ctx context.Context, cfg Config, _ limiter.Limiter) (*Local, error) { - return Open(ctx, cfg) - }) + return location.NewLimitedBackendFactory(ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open)) } const defaultLayout = "default" diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index f0a7ef9bc..1e12df808 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -44,11 +44,7 @@ type SFTP struct { var _ restic.Backend = &SFTP{} func NewFactory() location.Factory { - return location.NewLimitedBackendFactory(ParseConfig, location.NoPassword, func(ctx context.Context, cfg Config, _ limiter.Limiter) (*SFTP, error) { - return Create(ctx, cfg) - }, func(ctx context.Context, cfg Config, _ limiter.Limiter) (*SFTP, error) { - return Open(ctx, cfg) - }) + return location.NewLimitedBackendFactory(ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open)) } const defaultLayout = "default" From 555be49a7928c568877a4a60e5d92c3b701cb10f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 8 Jun 2023 15:05:07 +0200 Subject: [PATCH 31/90] location: Make ParseConfig-test backend agnostic The backend specific parts of the test are now directly handled by the respective backend. Duplicate tests were removed. --- internal/backend/local/config_test.go | 28 ++ internal/backend/location/location_test.go | 363 +++------------------ internal/backend/s3/config_test.go | 18 + 3 files changed, 87 insertions(+), 322 deletions(-) diff --git a/internal/backend/local/config_test.go b/internal/backend/local/config_test.go index c9b6be61c..4c2ebc7bc 100644 --- a/internal/backend/local/config_test.go +++ b/internal/backend/local/config_test.go @@ -11,6 +11,34 @@ var configTests = []test.ConfigTestData[Config]{ Path: "/some/path", Connections: 2, }}, + {S: "local:dir1/dir2", Cfg: Config{ + Path: "dir1/dir2", + Connections: 2, + }}, + {S: "local:../dir1/dir2", Cfg: Config{ + Path: "../dir1/dir2", + Connections: 2, + }}, + {S: "local:/dir1:foobar/dir2", Cfg: Config{ + Path: "/dir1:foobar/dir2", + Connections: 2, + }}, + {S: `local:\dir1\foobar\dir2`, Cfg: Config{ + Path: `\dir1\foobar\dir2`, + Connections: 2, + }}, + {S: `local:c:\dir1\foobar\dir2`, Cfg: Config{ + Path: `c:\dir1\foobar\dir2`, + Connections: 2, + }}, + {S: `local:C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`, Cfg: Config{ + Path: `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`, + Connections: 2, + }}, + {S: `local:c:/dir1/foobar/dir2`, Cfg: Config{ + Path: `c:/dir1/foobar/dir2`, + Connections: 2, + }}, } func TestParseConfig(t *testing.T) { diff --git a/internal/backend/location/location_test.go b/internal/backend/location/location_test.go index 6e9042200..933f2fc08 100644 --- a/internal/backend/location/location_test.go +++ b/internal/backend/location/location_test.go @@ -1,345 +1,64 @@ package location_test import ( - "net/url" - "reflect" "testing" - "github.com/restic/restic/internal/backend/b2" - "github.com/restic/restic/internal/backend/local" "github.com/restic/restic/internal/backend/location" - "github.com/restic/restic/internal/backend/rest" - "github.com/restic/restic/internal/backend/s3" - "github.com/restic/restic/internal/backend/sftp" - "github.com/restic/restic/internal/backend/swift" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" ) -func parseURL(s string) *url.URL { - u, err := url.Parse(s) - if err != nil { - panic(err) - } - - return u +type testConfig struct { + loc string } -var parseTests = []struct { - s string - u location.Location -}{ - { - "local:/srv/repo", - location.Location{Scheme: "local", - Config: &local.Config{ - Path: "/srv/repo", - Connections: 2, - }, - }, - }, - { - "local:dir1/dir2", - location.Location{Scheme: "local", - Config: &local.Config{ - Path: "dir1/dir2", - Connections: 2, - }, - }, - }, - { - "local:dir1/dir2", - location.Location{Scheme: "local", - Config: &local.Config{ - Path: "dir1/dir2", - Connections: 2, - }, - }, - }, - { - "dir1/dir2", - location.Location{Scheme: "local", - Config: &local.Config{ - Path: "dir1/dir2", - Connections: 2, - }, - }, - }, - { - "/dir1/dir2", - location.Location{Scheme: "local", - Config: &local.Config{ - Path: "/dir1/dir2", - Connections: 2, - }, - }, - }, - { - "local:../dir1/dir2", - location.Location{Scheme: "local", - Config: &local.Config{ - Path: "../dir1/dir2", - Connections: 2, - }, - }, - }, - { - "/dir1/dir2", - location.Location{Scheme: "local", - Config: &local.Config{ - Path: "/dir1/dir2", - Connections: 2, - }, - }, - }, - { - "/dir1:foobar/dir2", - location.Location{Scheme: "local", - Config: &local.Config{ - Path: "/dir1:foobar/dir2", - Connections: 2, - }, - }, - }, - { - `\dir1\foobar\dir2`, - location.Location{Scheme: "local", - Config: &local.Config{ - Path: `\dir1\foobar\dir2`, - Connections: 2, - }, - }, - }, - { - `c:\dir1\foobar\dir2`, - location.Location{Scheme: "local", - Config: &local.Config{ - Path: `c:\dir1\foobar\dir2`, - Connections: 2, - }, - }, - }, - { - `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`, - location.Location{Scheme: "local", - Config: &local.Config{ - Path: `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`, - Connections: 2, - }, - }, - }, - { - `c:/dir1/foobar/dir2`, - location.Location{Scheme: "local", - Config: &local.Config{ - Path: `c:/dir1/foobar/dir2`, - Connections: 2, - }, - }, - }, - { - "sftp:user@host:/srv/repo", - location.Location{Scheme: "sftp", - Config: &sftp.Config{ - User: "user", - Host: "host", - Path: "/srv/repo", - Connections: 5, - }, - }, - }, - { - "sftp:host:/srv/repo", - location.Location{Scheme: "sftp", - Config: &sftp.Config{ - User: "", - Host: "host", - Path: "/srv/repo", - Connections: 5, - }, - }, - }, - { - "sftp://user@host/srv/repo", - location.Location{Scheme: "sftp", - Config: &sftp.Config{ - User: "user", - Host: "host", - Path: "srv/repo", - Connections: 5, - }, - }, - }, - { - "sftp://user@host//srv/repo", - location.Location{Scheme: "sftp", - Config: &sftp.Config{ - User: "user", - Host: "host", - Path: "/srv/repo", - Connections: 5, - }, - }, - }, - - { - "s3://eu-central-1/bucketname", - location.Location{Scheme: "s3", - Config: &s3.Config{ - Endpoint: "eu-central-1", - Bucket: "bucketname", - Prefix: "", - Connections: 5, - }, - }, - }, - { - "s3://hostname.foo/bucketname", - location.Location{Scheme: "s3", - Config: &s3.Config{ - Endpoint: "hostname.foo", - Bucket: "bucketname", - Prefix: "", - Connections: 5, - }, - }, - }, - { - "s3://hostname.foo/bucketname/prefix/directory", - location.Location{Scheme: "s3", - Config: &s3.Config{ - Endpoint: "hostname.foo", - Bucket: "bucketname", - Prefix: "prefix/directory", - Connections: 5, - }, - }, - }, - { - "s3:eu-central-1/repo", - location.Location{Scheme: "s3", - Config: &s3.Config{ - Endpoint: "eu-central-1", - Bucket: "repo", - Prefix: "", - Connections: 5, - }, - }, - }, - { - "s3:eu-central-1/repo/prefix/directory", - location.Location{Scheme: "s3", - Config: &s3.Config{ - Endpoint: "eu-central-1", - Bucket: "repo", - Prefix: "prefix/directory", - Connections: 5, - }, - }, - }, - { - "s3:https://hostname.foo/repo", - location.Location{Scheme: "s3", - Config: &s3.Config{ - Endpoint: "hostname.foo", - Bucket: "repo", - Prefix: "", - Connections: 5, - }, - }, - }, - { - "s3:https://hostname.foo/repo/prefix/directory", - location.Location{Scheme: "s3", - Config: &s3.Config{ - Endpoint: "hostname.foo", - Bucket: "repo", - Prefix: "prefix/directory", - Connections: 5, - }, - }, - }, - { - "s3:http://hostname.foo/repo", - location.Location{Scheme: "s3", - Config: &s3.Config{ - Endpoint: "hostname.foo", - Bucket: "repo", - Prefix: "", - UseHTTP: true, - Connections: 5, - }, - }, - }, - { - "swift:container17:/", - location.Location{Scheme: "swift", - Config: &swift.Config{ - Container: "container17", - Prefix: "", - Connections: 5, - }, - }, - }, - { - "swift:container17:/prefix97", - location.Location{Scheme: "swift", - Config: &swift.Config{ - Container: "container17", - Prefix: "prefix97", - Connections: 5, - }, - }, - }, - { - "rest:http://hostname.foo:1234/", - location.Location{Scheme: "rest", - Config: &rest.Config{ - URL: parseURL("http://hostname.foo:1234/"), - Connections: 5, - }, - }, - }, - { - "b2:bucketname:/prefix", location.Location{Scheme: "b2", - Config: &b2.Config{ - Bucket: "bucketname", - Prefix: "prefix", - Connections: 5, - }, - }, - }, - { - "b2:bucketname", location.Location{Scheme: "b2", - Config: &b2.Config{ - Bucket: "bucketname", - Prefix: "", - Connections: 5, - }, - }, - }, +func testFactory() location.Factory { + return location.NewHTTPBackendFactory[testConfig, restic.Backend]( + func(s string) (*testConfig, error) { + return &testConfig{loc: s}, nil + }, nil, nil, nil, + ) } func TestParse(t *testing.T) { - for i, test := range parseTests { - t.Run(test.s, func(t *testing.T) { - u, err := location.Parse(test.s) + registry := location.NewRegistry() + registry.Register("test", testFactory()) + + path := "test:example" + u, err := location.Parse(registry, path) + test.OK(t, err) + test.Equals(t, "test", u.Scheme) + test.Equals(t, &testConfig{loc: path}, u.Config) +} + +func TestParseFallback(t *testing.T) { + fallbackTests := []string{ + "dir1/dir2", + "/dir1/dir2", + "/dir1:foobar/dir2", + `\dir1\foobar\dir2`, + `c:\dir1\foobar\dir2`, + `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`, + `c:/dir1/foobar/dir2`, + } + + registry := location.NewRegistry() + registry.Register("local", testFactory()) + + for _, path := range fallbackTests { + t.Run(path, func(t *testing.T) { + u, err := location.Parse(registry, path) if err != nil { t.Fatalf("unexpected error: %v", err) } - - if test.u.Scheme != u.Scheme { - t.Errorf("test %d: scheme does not match, want %q, got %q", - i, test.u.Scheme, u.Scheme) - } - - if !reflect.DeepEqual(test.u.Config, u.Config) { - t.Errorf("test %d: cfg map does not match, want:\n %#v\ngot: \n %#v", - i, test.u.Config, u.Config) - } + test.Equals(t, "local", u.Scheme) + test.Equals(t, "local:"+path, u.Config.(*testConfig).loc) }) } } func TestInvalidScheme(t *testing.T) { + registry := location.NewRegistry() var invalidSchemes = []string{ "foobar:xxx", "foobar:/dir/dir2", @@ -347,7 +66,7 @@ func TestInvalidScheme(t *testing.T) { for _, s := range invalidSchemes { t.Run(s, func(t *testing.T) { - _, err := location.Parse(s) + _, err := location.Parse(registry, s) if err == nil { t.Fatalf("error for invalid location %q not found", s) } diff --git a/internal/backend/s3/config_test.go b/internal/backend/s3/config_test.go index 21fbb27b9..085dbeedb 100644 --- a/internal/backend/s3/config_test.go +++ b/internal/backend/s3/config_test.go @@ -56,6 +56,24 @@ var configTests = []test.ConfigTestData[Config]{ Prefix: "prefix/directory", Connections: 5, }}, + {S: "s3:hostname.foo/foobar", Cfg: Config{ + Endpoint: "hostname.foo", + Bucket: "foobar", + Prefix: "", + Connections: 5, + }}, + {S: "s3:hostname.foo/foobar/prefix/directory", Cfg: Config{ + Endpoint: "hostname.foo", + Bucket: "foobar", + Prefix: "prefix/directory", + Connections: 5, + }}, + {S: "s3:https://hostname/foobar", Cfg: Config{ + Endpoint: "hostname", + Bucket: "foobar", + Prefix: "", + Connections: 5, + }}, {S: "s3:https://hostname:9999/foobar", Cfg: Config{ Endpoint: "hostname:9999", Bucket: "foobar", From 3325a7c8623ecf9fd39416baf61d9fbe2d00b051 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 8 Jun 2023 15:17:00 +0200 Subject: [PATCH 32/90] location: extract backend specific part of StripPassword The tests for the rest backend now reside there. --- .../backend/location/display_location_test.go | 115 ++++-------------- internal/backend/rest/config_test.go | 68 +++++++++++ 2 files changed, 92 insertions(+), 91 deletions(-) diff --git a/internal/backend/location/display_location_test.go b/internal/backend/location/display_location_test.go index 30d3cc286..4a4055a84 100644 --- a/internal/backend/location/display_location_test.go +++ b/internal/backend/location/display_location_test.go @@ -1,96 +1,29 @@ -package location +package location_test -import "testing" +import ( + "testing" -var passwordTests = []struct { - input string - expected string -}{ - { - "local:/srv/repo", - "local:/srv/repo", - }, - { - "/dir1/dir2", - "/dir1/dir2", - }, - { - `c:\dir1\foobar\dir2`, - `c:\dir1\foobar\dir2`, - }, - { - "sftp:user@host:/srv/repo", - "sftp:user@host:/srv/repo", - }, - { - "s3://eu-central-1/bucketname", - "s3://eu-central-1/bucketname", - }, - { - "swift:container17:/prefix97", - "swift:container17:/prefix97", - }, - { - "b2:bucketname:/prefix", - "b2:bucketname:/prefix", - }, - { - "rest:", - "rest:/", - }, - { - "rest:localhost/", - "rest:localhost/", - }, - { - "rest::123/", - "rest::123/", - }, - { - "rest:http://", - "rest:http://", - }, - { - "rest:http://hostname.foo:1234/", - "rest:http://hostname.foo:1234/", - }, - { - "rest:http://user@hostname.foo:1234/", - "rest:http://user@hostname.foo:1234/", - }, - { - "rest:http://user:@hostname.foo:1234/", - "rest:http://user:***@hostname.foo:1234/", - }, - { - "rest:http://user:p@hostname.foo:1234/", - "rest:http://user:***@hostname.foo:1234/", - }, - { - "rest:http://user:pppppaaafhhfuuwiiehhthhghhdkjaoowpprooghjjjdhhwuuhgjsjhhfdjhruuhsjsdhhfhshhsppwufhhsjjsjs@hostname.foo:1234/", - "rest:http://user:***@hostname.foo:1234/", - }, - { - "rest:http://user:password@hostname", - "rest:http://user:***@hostname/", - }, - { - "rest:http://user:password@:123", - "rest:http://user:***@:123/", - }, - { - "rest:http://user:password@", - "rest:http://user:***@/", - }, -} + "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" +) func TestStripPassword(t *testing.T) { - for i, test := range passwordTests { - t.Run(test.input, func(t *testing.T) { - result := StripPassword(test.input) - if result != test.expected { - t.Errorf("test %d: expected '%s' but got '%s'", i, test.expected, result) - } - }) - } + registry := location.NewRegistry() + registry.Register("test", + location.NewHTTPBackendFactory[any, restic.Backend](nil, + func(s string) string { + return "cleaned" + }, nil, nil, + ), + ) + + t.Run("valid", func(t *testing.T) { + clean := location.StripPassword(registry, "test:secret") + test.Equals(t, "cleaned", clean) + }) + t.Run("unknown", func(t *testing.T) { + clean := location.StripPassword(registry, "invalid:secret") + test.Equals(t, "invalid:secret", clean) + }) } diff --git a/internal/backend/rest/config_test.go b/internal/backend/rest/config_test.go index 8cfc78407..23ea9095b 100644 --- a/internal/backend/rest/config_test.go +++ b/internal/backend/rest/config_test.go @@ -36,3 +36,71 @@ var configTests = []test.ConfigTestData[Config]{ func TestParseConfig(t *testing.T) { test.ParseConfigTester(t, ParseConfig, configTests) } + +var passwordTests = []struct { + input string + expected string +}{ + { + "rest:", + "rest:/", + }, + { + "rest:localhost/", + "rest:localhost/", + }, + { + "rest::123/", + "rest::123/", + }, + { + "rest:http://", + "rest:http://", + }, + { + "rest:http://hostname.foo:1234/", + "rest:http://hostname.foo:1234/", + }, + { + "rest:http://user@hostname.foo:1234/", + "rest:http://user@hostname.foo:1234/", + }, + { + "rest:http://user:@hostname.foo:1234/", + "rest:http://user:***@hostname.foo:1234/", + }, + { + "rest:http://user:p@hostname.foo:1234/", + "rest:http://user:***@hostname.foo:1234/", + }, + { + "rest:http://user:pppppaaafhhfuuwiiehhthhghhdkjaoowpprooghjjjdhhwuuhgjsjhhfdjhruuhsjsdhhfhshhsppwufhhsjjsjs@hostname.foo:1234/", + "rest:http://user:***@hostname.foo:1234/", + }, + { + "rest:http://user:password@hostname", + "rest:http://user:***@hostname/", + }, + { + "rest:http://user:password@:123", + "rest:http://user:***@:123/", + }, + { + "rest:http://user:password@", + "rest:http://user:***@/", + }, +} + +func TestStripPassword(t *testing.T) { + // Make sure that the factory uses the correct method + StripPassword := NewFactory().StripPassword + + for i, test := range passwordTests { + t.Run(test.input, func(t *testing.T) { + result := StripPassword(test.input) + if result != test.expected { + t.Errorf("test %d: expected '%s' but got '%s'", i, test.expected, result) + } + }) + } +} From 19ac12d95b5226d75a484378be87ace1a093c812 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 8 Jun 2023 15:18:43 +0200 Subject: [PATCH 33/90] location: make genericBackendFactory private --- internal/backend/location/registry.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/backend/location/registry.go b/internal/backend/location/registry.go index 5d644dfb9..f15095590 100644 --- a/internal/backend/location/registry.go +++ b/internal/backend/location/registry.go @@ -36,35 +36,35 @@ type Factory interface { Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) } -type GenericBackendFactory[C any, T restic.Backend] struct { +type genericBackendFactory[C any, T restic.Backend] struct { parseConfigFn func(s string) (*C, error) stripPasswordFn func(s string) string createFn func(ctx context.Context, cfg C, rt http.RoundTripper, lim limiter.Limiter) (T, error) openFn func(ctx context.Context, cfg C, rt http.RoundTripper, lim limiter.Limiter) (T, error) } -func (f *GenericBackendFactory[C, T]) ParseConfig(s string) (interface{}, error) { +func (f *genericBackendFactory[C, T]) ParseConfig(s string) (interface{}, error) { return f.parseConfigFn(s) } -func (f *GenericBackendFactory[C, T]) StripPassword(s string) string { +func (f *genericBackendFactory[C, T]) StripPassword(s string) string { if f.stripPasswordFn != nil { return f.stripPasswordFn(s) } return s } -func (f *GenericBackendFactory[C, T]) Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) { +func (f *genericBackendFactory[C, T]) Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) { return f.createFn(ctx, *cfg.(*C), rt, lim) } -func (f *GenericBackendFactory[C, T]) Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) { +func (f *genericBackendFactory[C, T]) Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) { return f.openFn(ctx, *cfg.(*C), rt, lim) } func NewHTTPBackendFactory[C any, T restic.Backend](parseConfigFn func(s string) (*C, error), stripPasswordFn func(s string) string, createFn func(ctx context.Context, cfg C, rt http.RoundTripper) (T, error), - openFn func(ctx context.Context, cfg C, rt http.RoundTripper) (T, error)) *GenericBackendFactory[C, T] { + openFn func(ctx context.Context, cfg C, rt http.RoundTripper) (T, error)) Factory { - return &GenericBackendFactory[C, T]{ + return &genericBackendFactory[C, T]{ parseConfigFn: parseConfigFn, stripPasswordFn: stripPasswordFn, createFn: func(ctx context.Context, cfg C, rt http.RoundTripper, _ limiter.Limiter) (T, error) { @@ -79,9 +79,9 @@ func NewHTTPBackendFactory[C any, T restic.Backend](parseConfigFn func(s string) func NewLimitedBackendFactory[C any, T restic.Backend](parseConfigFn func(s string) (*C, error), stripPasswordFn func(s string) string, createFn func(ctx context.Context, cfg C, lim limiter.Limiter) (T, error), - openFn func(ctx context.Context, cfg C, lim limiter.Limiter) (T, error)) *GenericBackendFactory[C, T] { + openFn func(ctx context.Context, cfg C, lim limiter.Limiter) (T, error)) Factory { - return &GenericBackendFactory[C, T]{ + return &genericBackendFactory[C, T]{ parseConfigFn: parseConfigFn, stripPasswordFn: stripPasswordFn, createFn: func(ctx context.Context, cfg C, _ http.RoundTripper, lim limiter.Limiter) (T, error) { From 3a3cf608f5bbf59b0a72e9796141dba09032cdec Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 8 Jun 2023 15:28:07 +0200 Subject: [PATCH 34/90] b2/s3: Move config validation from ApplyEnvironment to Open/Create Conceptually the backend configuration should be validated when creating or opening the backend, but not when filling in information from environment variables into the configuration. --- cmd/restic/global.go | 4 +--- internal/backend/azure/azure_test.go | 6 +----- internal/backend/azure/config.go | 3 +-- internal/backend/b2/b2.go | 7 +++++++ internal/backend/b2/b2_test.go | 6 +----- internal/backend/b2/config.go | 12 +----------- internal/backend/gs/config.go | 3 +-- internal/backend/s3/config.go | 12 +----------- internal/backend/s3/s3.go | 6 ++++++ internal/backend/swift/config.go | 3 +-- internal/backend/swift/swift_test.go | 4 +--- internal/restic/backend.go | 2 +- 12 files changed, 23 insertions(+), 45 deletions(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index f08a75b13..1c13fb887 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -550,9 +550,7 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi func parseConfig(loc location.Location, opts options.Options) (interface{}, error) { cfg := loc.Config if cfg, ok := cfg.(restic.ApplyEnvironmenter); ok { - if err := cfg.ApplyEnvironment(""); err != nil { - return nil, err - } + cfg.ApplyEnvironment("") } // only apply options for a particular backend here diff --git a/internal/backend/azure/azure_test.go b/internal/backend/azure/azure_test.go index 0fab5da26..5aee96fbd 100644 --- a/internal/backend/azure/azure_test.go +++ b/internal/backend/azure/azure_test.go @@ -35,11 +35,7 @@ func newAzureTestSuite(t testing.TB) *test.Suite[azure.Config] { return nil, err } - err = cfg.ApplyEnvironment("RESTIC_TEST_") - if err != nil { - return nil, err - } - + cfg.ApplyEnvironment("RESTIC_TEST_") cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) return cfg, nil }, diff --git a/internal/backend/azure/config.go b/internal/backend/azure/config.go index 4d4e839ff..6786ec626 100644 --- a/internal/backend/azure/config.go +++ b/internal/backend/azure/config.go @@ -59,7 +59,7 @@ func ParseConfig(s string) (*Config, error) { var _ restic.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. -func (cfg *Config) ApplyEnvironment(prefix string) error { +func (cfg *Config) ApplyEnvironment(prefix string) { if cfg.AccountName == "" { cfg.AccountName = os.Getenv(prefix + "AZURE_ACCOUNT_NAME") } @@ -71,5 +71,4 @@ func (cfg *Config) ApplyEnvironment(prefix string) error { if cfg.AccountSAS.String() == "" { cfg.AccountSAS = options.NewSecretString(os.Getenv(prefix + "AZURE_ACCOUNT_SAS")) } - return nil } diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go index eb2cfe3c2..3560cca49 100644 --- a/internal/backend/b2/b2.go +++ b/internal/backend/b2/b2.go @@ -58,6 +58,13 @@ func (s *sniffingRoundTripper) RoundTrip(req *http.Request) (*http.Response, err } func newClient(ctx context.Context, cfg Config, rt http.RoundTripper) (*b2.Client, error) { + if cfg.AccountID == "" { + return nil, errors.Fatalf("unable to open B2 backend: Account ID ($B2_ACCOUNT_ID) is empty") + } + if cfg.Key.String() == "" { + return nil, errors.Fatalf("unable to open B2 backend: Key ($B2_ACCOUNT_KEY) is empty") + } + sniffer := &sniffingRoundTripper{RoundTripper: rt} opts := []b2.ClientOption{b2.Transport(sniffer)} diff --git a/internal/backend/b2/b2_test.go b/internal/backend/b2/b2_test.go index 8e982adda..3a649db1e 100644 --- a/internal/backend/b2/b2_test.go +++ b/internal/backend/b2/b2_test.go @@ -35,11 +35,7 @@ func newB2TestSuite(t testing.TB) *test.Suite[b2.Config] { return nil, err } - err = cfg.ApplyEnvironment("RESTIC_TEST_") - if err != nil { - return nil, err - } - + cfg.ApplyEnvironment("RESTIC_TEST_") cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) return cfg, nil }, diff --git a/internal/backend/b2/config.go b/internal/backend/b2/config.go index 548fbef99..94614e44f 100644 --- a/internal/backend/b2/config.go +++ b/internal/backend/b2/config.go @@ -85,21 +85,11 @@ func ParseConfig(s string) (*Config, error) { var _ restic.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. -func (cfg *Config) ApplyEnvironment(prefix string) error { +func (cfg *Config) ApplyEnvironment(prefix string) { if cfg.AccountID == "" { cfg.AccountID = os.Getenv(prefix + "B2_ACCOUNT_ID") } - - if cfg.AccountID == "" { - return errors.Fatalf("unable to open B2 backend: Account ID ($B2_ACCOUNT_ID) is empty") - } - if cfg.Key.String() == "" { cfg.Key = options.NewSecretString(os.Getenv(prefix + "B2_ACCOUNT_KEY")) } - - if cfg.Key.String() == "" { - return errors.Fatalf("unable to open B2 backend: Key ($B2_ACCOUNT_KEY) is empty") - } - return nil } diff --git a/internal/backend/gs/config.go b/internal/backend/gs/config.go index b2d52c5f8..61a31113f 100644 --- a/internal/backend/gs/config.go +++ b/internal/backend/gs/config.go @@ -62,9 +62,8 @@ func ParseConfig(s string) (*Config, error) { var _ restic.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. -func (cfg *Config) ApplyEnvironment(prefix string) error { +func (cfg *Config) ApplyEnvironment(prefix string) { if cfg.ProjectID == "" { cfg.ProjectID = os.Getenv(prefix + "GOOGLE_PROJECT_ID") } - return nil } diff --git a/internal/backend/s3/config.go b/internal/backend/s3/config.go index 525373d16..8dcad9eee 100644 --- a/internal/backend/s3/config.go +++ b/internal/backend/s3/config.go @@ -97,24 +97,14 @@ func createConfig(endpoint, bucket, prefix string, useHTTP bool) (*Config, error var _ restic.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. -func (cfg *Config) ApplyEnvironment(prefix string) error { +func (cfg *Config) ApplyEnvironment(prefix string) { if cfg.KeyID == "" { cfg.KeyID = os.Getenv(prefix + "AWS_ACCESS_KEY_ID") } - if cfg.Secret.String() == "" { cfg.Secret = options.NewSecretString(os.Getenv(prefix + "AWS_SECRET_ACCESS_KEY")) } - - if cfg.KeyID == "" && cfg.Secret.String() != "" { - return errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty") - } else if cfg.KeyID != "" && cfg.Secret.String() == "" { - return errors.Fatalf("unable to open S3 backend: Secret ($AWS_SECRET_ACCESS_KEY) is empty") - } - if cfg.Region == "" { cfg.Region = os.Getenv(prefix + "AWS_DEFAULT_REGION") } - - return nil } diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index dd5cc36e6..10512e809 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -41,6 +41,12 @@ const defaultLayout = "default" func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) { debug.Log("open, config %#v", cfg) + if cfg.KeyID == "" && cfg.Secret.String() != "" { + return nil, errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty") + } else if cfg.KeyID != "" && cfg.Secret.String() == "" { + return nil, errors.Fatalf("unable to open S3 backend: Secret ($AWS_SECRET_ACCESS_KEY) is empty") + } + if cfg.MaxRetries > 0 { minio.MaxRetry = int(cfg.MaxRetries) } diff --git a/internal/backend/swift/config.go b/internal/backend/swift/config.go index b9f5d3995..5be2d9ce0 100644 --- a/internal/backend/swift/config.go +++ b/internal/backend/swift/config.go @@ -77,7 +77,7 @@ func ParseConfig(s string) (*Config, error) { var _ restic.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. -func (cfg *Config) ApplyEnvironment(prefix string) error { +func (cfg *Config) ApplyEnvironment(prefix string) { for _, val := range []struct { s *string env string @@ -130,5 +130,4 @@ func (cfg *Config) ApplyEnvironment(prefix string) error { *val.s = options.NewSecretString(os.Getenv(val.env)) } } - return nil } diff --git a/internal/backend/swift/swift_test.go b/internal/backend/swift/swift_test.go index cb0992010..52278943e 100644 --- a/internal/backend/swift/swift_test.go +++ b/internal/backend/swift/swift_test.go @@ -48,9 +48,7 @@ func newSwiftTestSuite(t testing.TB) *test.Suite[swift.Config] { return nil, err } - if err = cfg.ApplyEnvironment("RESTIC_TEST_"); err != nil { - return nil, err - } + cfg.ApplyEnvironment("RESTIC_TEST_") cfg.Prefix += fmt.Sprintf("/test-%d", time.Now().UnixNano()) t.Logf("using prefix %v", cfg.Prefix) return cfg, nil diff --git a/internal/restic/backend.go b/internal/restic/backend.go index b6653fcb4..555b9d96e 100644 --- a/internal/restic/backend.go +++ b/internal/restic/backend.go @@ -83,5 +83,5 @@ type FileInfo struct { // ApplyEnvironmenter fills in a backend configuration from the environment type ApplyEnvironmenter interface { - ApplyEnvironment(prefix string) error + ApplyEnvironment(prefix string) } From 3d3bb887453d84ef8561e5c8e97b17c9dc51a8e8 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 8 Jun 2023 16:21:59 +0200 Subject: [PATCH 35/90] b2: remove duplicate check for config file during repository creation No other backend implements that check. The check that a repository is not yet initialized is handled by the Repository later on. --- internal/backend/b2/b2.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go index 3560cca49..700fff099 100644 --- a/internal/backend/b2/b2.go +++ b/internal/backend/b2/b2.go @@ -147,16 +147,6 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backe }, listMaxItems: defaultListMaxItems, } - - _, err = be.Stat(ctx, restic.Handle{Type: restic.ConfigFile}) - if err != nil && !be.IsNotExist(err) { - return nil, err - } - - if err == nil { - return nil, errors.New("config already exists") - } - return be, nil } From 13a8b5822f1115ca11f6d15e47a3a6a61e839aa5 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 8 Jun 2023 16:53:55 +0200 Subject: [PATCH 36/90] backend: Adjust tests to use the Factory to instantiate the backend This drastically reduces the amount of duplicated test code. --- internal/backend/azure/azure_test.go | 43 +------ internal/backend/b2/b2_test.go | 28 +---- internal/backend/gs/gs_test.go | 43 +------ internal/backend/local/local_test.go | 27 +---- internal/backend/mem/mem_backend.go | 20 +++ internal/backend/mem/mem_backend_test.go | 48 +------- internal/backend/rclone/backend_test.go | 29 ++--- internal/backend/rest/rest_test.go | 22 +--- internal/backend/s3/s3_test.go | 147 +++++------------------ internal/backend/sftp/sftp_test.go | 28 +---- internal/backend/swift/swift_test.go | 42 +------ internal/backend/test/suite.go | 63 +++++++--- internal/backend/test/tests.go | 2 +- 13 files changed, 119 insertions(+), 423 deletions(-) diff --git a/internal/backend/azure/azure_test.go b/internal/backend/azure/azure_test.go index 5aee96fbd..8465cc3b0 100644 --- a/internal/backend/azure/azure_test.go +++ b/internal/backend/azure/azure_test.go @@ -12,18 +12,12 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/azure" "github.com/restic/restic/internal/backend/test" - "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) func newAzureTestSuite(t testing.TB) *test.Suite[azure.Config] { - tr, err := backend.Transport(backend.TransportOptions{}) - if err != nil { - t.Fatalf("cannot create transport for tests: %v", err) - } - return &test.Suite[azure.Config]{ // do not use excessive data MinimalData: true, @@ -40,42 +34,7 @@ func newAzureTestSuite(t testing.TB) *test.Suite[azure.Config] { return cfg, nil }, - // CreateFn is a function that creates a temporary repository for the tests. - Create: func(cfg azure.Config) (restic.Backend, error) { - ctx := context.TODO() - be, err := azure.Create(ctx, cfg, tr) - if err != nil { - return nil, err - } - - _, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile}) - if err != nil && !be.IsNotExist(err) { - return nil, err - } - - if err == nil { - return nil, errors.New("config already exists") - } - - return be, nil - }, - - // OpenFn is a function that opens a previously created temporary repository. - Open: func(cfg azure.Config) (restic.Backend, error) { - ctx := context.TODO() - return azure.Open(ctx, cfg, tr) - }, - - // CleanupFn removes data created during the tests. - Cleanup: func(cfg azure.Config) error { - ctx := context.TODO() - be, err := azure.Open(ctx, cfg, tr) - if err != nil { - return err - } - - return be.Delete(context.TODO()) - }, + Factory: azure.NewFactory(), } } diff --git a/internal/backend/b2/b2_test.go b/internal/backend/b2/b2_test.go index 3a649db1e..348af9095 100644 --- a/internal/backend/b2/b2_test.go +++ b/internal/backend/b2/b2_test.go @@ -1,26 +1,18 @@ package b2_test import ( - "context" "fmt" "os" "testing" "time" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/b2" "github.com/restic/restic/internal/backend/test" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) func newB2TestSuite(t testing.TB) *test.Suite[b2.Config] { - tr, err := backend.Transport(backend.TransportOptions{}) - if err != nil { - t.Fatalf("cannot create transport for tests: %v", err) - } - return &test.Suite[b2.Config]{ // do not use excessive data MinimalData: true, @@ -40,25 +32,7 @@ func newB2TestSuite(t testing.TB) *test.Suite[b2.Config] { return cfg, nil }, - // CreateFn is a function that creates a temporary repository for the tests. - Create: func(cfg b2.Config) (restic.Backend, error) { - return b2.Create(context.Background(), cfg, tr) - }, - - // OpenFn is a function that opens a previously created temporary repository. - Open: func(cfg b2.Config) (restic.Backend, error) { - return b2.Open(context.Background(), cfg, tr) - }, - - // CleanupFn removes data created during the tests. - Cleanup: func(cfg b2.Config) error { - be, err := b2.Open(context.Background(), cfg, tr) - if err != nil { - return err - } - - return be.Delete(context.TODO()) - }, + Factory: b2.NewFactory(), } } diff --git a/internal/backend/gs/gs_test.go b/internal/backend/gs/gs_test.go index a73fe77fb..47085dc4e 100644 --- a/internal/backend/gs/gs_test.go +++ b/internal/backend/gs/gs_test.go @@ -1,26 +1,17 @@ package gs_test import ( - "context" "fmt" "os" "testing" "time" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/gs" "github.com/restic/restic/internal/backend/test" - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) func newGSTestSuite(t testing.TB) *test.Suite[gs.Config] { - tr, err := backend.Transport(backend.TransportOptions{}) - if err != nil { - t.Fatalf("cannot create transport for tests: %v", err) - } - return &test.Suite[gs.Config]{ // do not use excessive data MinimalData: true, @@ -37,39 +28,7 @@ func newGSTestSuite(t testing.TB) *test.Suite[gs.Config] { return cfg, nil }, - // CreateFn is a function that creates a temporary repository for the tests. - Create: func(cfg gs.Config) (restic.Backend, error) { - be, err := gs.Create(context.Background(), cfg, tr) - if err != nil { - return nil, err - } - - _, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile}) - if err != nil && !be.IsNotExist(err) { - return nil, err - } - - if err == nil { - return nil, errors.New("config already exists") - } - - return be, nil - }, - - // OpenFn is a function that opens a previously created temporary repository. - Open: func(cfg gs.Config) (restic.Backend, error) { - return gs.Open(context.TODO(), cfg, tr) - }, - - // CleanupFn removes data created during the tests. - Cleanup: func(cfg gs.Config) error { - be, err := gs.Open(context.TODO(), cfg, tr) - if err != nil { - return err - } - - return be.Delete(context.TODO()) - }, + Factory: gs.NewFactory(), } } diff --git a/internal/backend/local/local_test.go b/internal/backend/local/local_test.go index ca9e3b71b..2a8b626d4 100644 --- a/internal/backend/local/local_test.go +++ b/internal/backend/local/local_test.go @@ -8,7 +8,6 @@ import ( "github.com/restic/restic/internal/backend/local" "github.com/restic/restic/internal/backend/test" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -16,11 +15,7 @@ func newTestSuite(t testing.TB) *test.Suite[local.Config] { return &test.Suite[local.Config]{ // NewConfig returns a config for a new temporary backend that will be used in tests. NewConfig: func() (*local.Config, error) { - dir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-local-") - if err != nil { - t.Fatal(err) - } - + dir := rtest.TempDir(t) t.Logf("create new backend at %v", dir) cfg := &local.Config{ @@ -30,25 +25,7 @@ func newTestSuite(t testing.TB) *test.Suite[local.Config] { return cfg, nil }, - // CreateFn is a function that creates a temporary repository for the tests. - Create: func(cfg local.Config) (restic.Backend, error) { - return local.Create(context.TODO(), cfg) - }, - - // OpenFn is a function that opens a previously created temporary repository. - Open: func(cfg local.Config) (restic.Backend, error) { - return local.Open(context.TODO(), cfg) - }, - - // CleanupFn removes data created during the tests. - Cleanup: func(cfg local.Config) error { - if !rtest.TestCleanupTempDirs { - t.Logf("leaving test backend dir at %v", cfg.Path) - } - - rtest.RemoveAll(t, cfg.Path) - return nil - }, + Factory: local.NewFactory(), } } diff --git a/internal/backend/mem/mem_backend.go b/internal/backend/mem/mem_backend.go index 618ef5752..a467d33f7 100644 --- a/internal/backend/mem/mem_backend.go +++ b/internal/backend/mem/mem_backend.go @@ -6,10 +6,12 @@ import ( "encoding/base64" "hash" "io" + "net/http" "sync" "github.com/cespare/xxhash/v2" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -20,6 +22,24 @@ type memMap map[restic.Handle][]byte // make sure that MemoryBackend implements backend.Backend var _ restic.Backend = &MemoryBackend{} +// NewFactory creates a persistent mem backend +func NewFactory() location.Factory { + be := New() + + return location.NewHTTPBackendFactory[struct{}, *MemoryBackend]( + func(s string) (*struct{}, error) { + return &struct{}{}, nil + }, + location.NoPassword, + func(_ context.Context, _ struct{}, _ http.RoundTripper) (*MemoryBackend, error) { + return be, nil + }, + func(_ context.Context, _ struct{}, _ http.RoundTripper) (*MemoryBackend, error) { + return be, nil + }, + ) +} + var errNotFound = errors.New("not found") const connectionCount = 2 diff --git a/internal/backend/mem/mem_backend_test.go b/internal/backend/mem/mem_backend_test.go index 3dea089bc..c4dad0fb2 100644 --- a/internal/backend/mem/mem_backend_test.go +++ b/internal/backend/mem/mem_backend_test.go @@ -1,58 +1,20 @@ package mem_test import ( - "context" "testing" - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/backend/test" ) -type memConfig struct { - be restic.Backend -} - -func newTestSuite() *test.Suite[*memConfig] { - return &test.Suite[*memConfig]{ +func newTestSuite() *test.Suite[struct{}] { + return &test.Suite[struct{}]{ // NewConfig returns a config for a new temporary backend that will be used in tests. - NewConfig: func() (**memConfig, error) { - cfg := &memConfig{} - return &cfg, nil + NewConfig: func() (*struct{}, error) { + return &struct{}{}, nil }, - // CreateFn is a function that creates a temporary repository for the tests. - Create: func(cfg *memConfig) (restic.Backend, error) { - if cfg.be != nil { - _, err := cfg.be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile}) - if err != nil && !cfg.be.IsNotExist(err) { - return nil, err - } - - if err == nil { - return nil, errors.New("config already exists") - } - } - - cfg.be = mem.New() - return cfg.be, nil - }, - - // OpenFn is a function that opens a previously created temporary repository. - Open: func(cfg *memConfig) (restic.Backend, error) { - if cfg.be == nil { - cfg.be = mem.New() - } - return cfg.be, nil - }, - - // CleanupFn removes data created during the tests. - Cleanup: func(cfg *memConfig) error { - // no cleanup needed - return nil - }, + Factory: mem.NewFactory(), } } diff --git a/internal/backend/rclone/backend_test.go b/internal/backend/rclone/backend_test.go index 738462577..742031585 100644 --- a/internal/backend/rclone/backend_test.go +++ b/internal/backend/rclone/backend_test.go @@ -1,14 +1,11 @@ package rclone_test import ( - "context" "os/exec" "testing" "github.com/restic/restic/internal/backend/rclone" "github.com/restic/restic/internal/backend/test" - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -24,23 +21,15 @@ func newTestSuite(t testing.TB) *test.Suite[rclone.Config] { return &cfg, nil }, - // CreateFn is a function that creates a temporary repository for the tests. - Create: func(cfg rclone.Config) (restic.Backend, error) { - t.Logf("Create()") - be, err := rclone.Create(context.TODO(), cfg, nil) - var e *exec.Error - if errors.As(err, &e) && e.Err == exec.ErrNotFound { - t.Skipf("program %q not found", e.Name) - return nil, nil - } - return be, err - }, + Factory: rclone.NewFactory(), + } +} - // OpenFn is a function that opens a previously created temporary repository. - Open: func(cfg rclone.Config) (restic.Backend, error) { - t.Logf("Open()") - return rclone.Open(context.TODO(), cfg, nil) - }, +func findRclone(t testing.TB) { + // try to find a rclone binary + _, err := exec.LookPath("rclone") + if err != nil { + t.Skip(err) } } @@ -51,9 +40,11 @@ func TestBackendRclone(t *testing.T) { } }() + findRclone(t) newTestSuite(t).RunTests(t) } func BenchmarkBackendREST(t *testing.B) { + findRclone(t) newTestSuite(t).RunBenchmarks(t) } diff --git a/internal/backend/rest/rest_test.go b/internal/backend/rest/rest_test.go index 4d069e63c..60cc40afe 100644 --- a/internal/backend/rest/rest_test.go +++ b/internal/backend/rest/rest_test.go @@ -9,10 +9,8 @@ import ( "testing" "time" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/rest" "github.com/restic/restic/internal/backend/test" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -68,11 +66,6 @@ func runRESTServer(ctx context.Context, t testing.TB, dir string) (*url.URL, fun } func newTestSuite(_ context.Context, t testing.TB, url *url.URL, minimalData bool) *test.Suite[rest.Config] { - tr, err := backend.Transport(backend.TransportOptions{}) - if err != nil { - t.Fatalf("cannot create transport for tests: %v", err) - } - return &test.Suite[rest.Config]{ MinimalData: minimalData, @@ -83,20 +76,7 @@ func newTestSuite(_ context.Context, t testing.TB, url *url.URL, minimalData boo return &cfg, nil }, - // CreateFn is a function that creates a temporary repository for the tests. - Create: func(cfg rest.Config) (restic.Backend, error) { - return rest.Create(context.TODO(), cfg, tr) - }, - - // OpenFn is a function that opens a previously created temporary repository. - Open: func(cfg rest.Config) (restic.Backend, error) { - return rest.Open(context.TODO(), cfg, tr) - }, - - // CleanupFn removes data created during the tests. - Cleanup: func(cfg rest.Config) error { - return nil - }, + Factory: rest.NewFactory(), } } diff --git a/internal/backend/s3/s3_test.go b/internal/backend/s3/s3_test.go index 1cdc6d7e9..e645fe03e 100644 --- a/internal/backend/s3/s3_test.go +++ b/internal/backend/s3/s3_test.go @@ -4,22 +4,18 @@ import ( "context" "crypto/rand" "encoding/hex" - "errors" "fmt" "io" "net" - "net/http" "os" "os/exec" "path/filepath" "testing" "time" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/backend/test" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -98,84 +94,34 @@ func newRandomCredentials(t testing.TB) (key, secret string) { return key, secret } -type MinioTestConfig struct { - s3.Config - - tempdir string - stopServer func() -} - -func createS3(t testing.TB, cfg MinioTestConfig, tr http.RoundTripper) (be restic.Backend, err error) { - for i := 0; i < 10; i++ { - be, err = s3.Create(context.TODO(), cfg.Config, tr) - if err != nil { - t.Logf("s3 open: try %d: error %v", i, err) - time.Sleep(500 * time.Millisecond) - continue - } - - break - } - - return be, err -} - -func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite[MinioTestConfig] { - tr, err := backend.Transport(backend.TransportOptions{}) - if err != nil { - t.Fatalf("cannot create transport for tests: %v", err) - } - - return &test.Suite[MinioTestConfig]{ +func newMinioTestSuite(ctx context.Context, t testing.TB, key string, secret string) *test.Suite[s3.Config] { + return &test.Suite[s3.Config]{ // NewConfig returns a config for a new temporary backend that will be used in tests. - NewConfig: func() (*MinioTestConfig, error) { - cfg := MinioTestConfig{} - - cfg.tempdir = rtest.TempDir(t) - key, secret := newRandomCredentials(t) - cfg.stopServer = runMinio(ctx, t, cfg.tempdir, key, secret) - - cfg.Config = s3.NewConfig() - cfg.Config.Endpoint = "localhost:9000" - cfg.Config.Bucket = "restictestbucket" - cfg.Config.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) - cfg.Config.UseHTTP = true - cfg.Config.KeyID = key - cfg.Config.Secret = options.NewSecretString(secret) + NewConfig: func() (*s3.Config, error) { + cfg := s3.NewConfig() + cfg.Endpoint = "localhost:9000" + cfg.Bucket = "restictestbucket" + cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) + cfg.UseHTTP = true + cfg.KeyID = key + cfg.Secret = options.NewSecretString(secret) return &cfg, nil }, - // CreateFn is a function that creates a temporary repository for the tests. - Create: func(cfg MinioTestConfig) (restic.Backend, error) { - be, err := createS3(t, cfg, tr) - if err != nil { - return nil, err - } + Factory: s3.NewFactory(), + } +} - _, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile}) - if err != nil && !be.IsNotExist(err) { - return nil, err - } +func createMinioTestSuite(t testing.TB) (*test.Suite[s3.Config], func()) { + ctx, cancel := context.WithCancel(context.Background()) - if err == nil { - return nil, errors.New("config already exists") - } + tempdir := rtest.TempDir(t) + key, secret := newRandomCredentials(t) + cleanup := runMinio(ctx, t, tempdir, key, secret) - return be, nil - }, - - // OpenFn is a function that opens a previously created temporary repository. - Open: func(cfg MinioTestConfig) (restic.Backend, error) { - return s3.Open(ctx, cfg.Config, tr) - }, - - // CleanupFn removes data created during the tests. - Cleanup: func(cfg MinioTestConfig) error { - if cfg.stopServer != nil { - cfg.stopServer() - } - return nil - }, + return newMinioTestSuite(ctx, t, key, secret), func() { + defer cancel() + defer cleanup() } } @@ -193,10 +139,10 @@ func TestBackendMinio(t *testing.T) { return } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + suite, cleanup := createMinioTestSuite(t) + defer cleanup() - newMinioTestSuite(ctx, t).RunTests(t) + suite.RunTests(t) } func BenchmarkBackendMinio(t *testing.B) { @@ -207,18 +153,13 @@ func BenchmarkBackendMinio(t *testing.B) { return } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + suite, cleanup := createMinioTestSuite(t) + defer cleanup() - newMinioTestSuite(ctx, t).RunBenchmarks(t) + suite.RunBenchmarks(t) } func newS3TestSuite(t testing.TB) *test.Suite[s3.Config] { - tr, err := backend.Transport(backend.TransportOptions{}) - if err != nil { - t.Fatalf("cannot create transport for tests: %v", err) - } - return &test.Suite[s3.Config]{ // do not use excessive data MinimalData: true, @@ -236,39 +177,7 @@ func newS3TestSuite(t testing.TB) *test.Suite[s3.Config] { return cfg, nil }, - // CreateFn is a function that creates a temporary repository for the tests. - Create: func(cfg s3.Config) (restic.Backend, error) { - be, err := s3.Create(context.TODO(), cfg, tr) - if err != nil { - return nil, err - } - - _, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile}) - if err != nil && !be.IsNotExist(err) { - return nil, err - } - - if err == nil { - return nil, errors.New("config already exists") - } - - return be, nil - }, - - // OpenFn is a function that opens a previously created temporary repository. - Open: func(cfg s3.Config) (restic.Backend, error) { - return s3.Open(context.TODO(), cfg, tr) - }, - - // CleanupFn removes data created during the tests. - Cleanup: func(cfg s3.Config) error { - be, err := s3.Open(context.TODO(), cfg, tr) - if err != nil { - return err - } - - return be.Delete(context.TODO()) - }, + Factory: s3.NewFactory(), } } diff --git a/internal/backend/sftp/sftp_test.go b/internal/backend/sftp/sftp_test.go index 98175ca26..75adc0c6b 100644 --- a/internal/backend/sftp/sftp_test.go +++ b/internal/backend/sftp/sftp_test.go @@ -1,7 +1,6 @@ package sftp_test import ( - "context" "fmt" "os" "path/filepath" @@ -11,7 +10,6 @@ import ( "github.com/restic/restic/internal/backend/sftp" "github.com/restic/restic/internal/backend/test" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -33,11 +31,7 @@ func newTestSuite(t testing.TB) *test.Suite[sftp.Config] { return &test.Suite[sftp.Config]{ // NewConfig returns a config for a new temporary backend that will be used in tests. NewConfig: func() (*sftp.Config, error) { - dir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-sftp-") - if err != nil { - t.Fatal(err) - } - + dir := rtest.TempDir(t) t.Logf("create new backend at %v", dir) cfg := &sftp.Config{ @@ -48,25 +42,7 @@ func newTestSuite(t testing.TB) *test.Suite[sftp.Config] { return cfg, nil }, - // CreateFn is a function that creates a temporary repository for the tests. - Create: func(cfg sftp.Config) (restic.Backend, error) { - return sftp.Create(context.TODO(), cfg) - }, - - // OpenFn is a function that opens a previously created temporary repository. - Open: func(cfg sftp.Config) (restic.Backend, error) { - return sftp.Open(context.TODO(), cfg) - }, - - // CleanupFn removes data created during the tests. - Cleanup: func(cfg sftp.Config) error { - if !rtest.TestCleanupTempDirs { - t.Logf("leaving test backend dir at %v", cfg.Path) - } - - rtest.RemoveAll(t, cfg.Path) - return nil - }, + Factory: sftp.NewFactory(), } } diff --git a/internal/backend/swift/swift_test.go b/internal/backend/swift/swift_test.go index 52278943e..98ee5b1c1 100644 --- a/internal/backend/swift/swift_test.go +++ b/internal/backend/swift/swift_test.go @@ -1,26 +1,18 @@ package swift_test import ( - "context" "fmt" "os" "testing" "time" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/swift" "github.com/restic/restic/internal/backend/test" - "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) func newSwiftTestSuite(t testing.TB) *test.Suite[swift.Config] { - tr, err := backend.Transport(backend.TransportOptions{}) - if err != nil { - t.Fatalf("cannot create transport for tests: %v", err) - } - return &test.Suite[swift.Config]{ // do not use excessive data MinimalData: true, @@ -54,39 +46,7 @@ func newSwiftTestSuite(t testing.TB) *test.Suite[swift.Config] { return cfg, nil }, - // CreateFn is a function that creates a temporary repository for the tests. - Create: func(cfg swift.Config) (restic.Backend, error) { - be, err := swift.Open(context.TODO(), cfg, tr) - if err != nil { - return nil, err - } - - _, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile}) - if err != nil && !be.IsNotExist(err) { - return nil, err - } - - if err == nil { - return nil, errors.New("config already exists") - } - - return be, nil - }, - - // OpenFn is a function that opens a previously created temporary repository. - Open: func(cfg swift.Config) (restic.Backend, error) { - return swift.Open(context.TODO(), cfg, tr) - }, - - // CleanupFn removes data created during the tests. - Cleanup: func(cfg swift.Config) error { - be, err := swift.Open(context.TODO(), cfg, tr) - if err != nil { - return err - } - - return be.Delete(context.TODO()) - }, + Factory: swift.NewFactory(), } } diff --git a/internal/backend/test/suite.go b/internal/backend/test/suite.go index 75ae0630b..bb77124d7 100644 --- a/internal/backend/test/suite.go +++ b/internal/backend/test/suite.go @@ -1,11 +1,16 @@ package test import ( + "context" + "fmt" "reflect" "strings" "testing" "time" + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) @@ -18,14 +23,8 @@ type Suite[C any] struct { // NewConfig returns a config for a new temporary backend that will be used in tests. NewConfig func() (*C, error) - // CreateFn is a function that creates a temporary repository for the tests. - Create func(cfg C) (restic.Backend, error) - - // OpenFn is a function that opens a previously created temporary repository. - Open func(cfg C) (restic.Backend, error) - - // CleanupFn removes data created during the tests. - Cleanup func(cfg C) error + // Factory contains a factory that can be used to create or open a repository for the tests. + Factory location.Factory // MinimalData instructs the tests to not use excessive data. MinimalData bool @@ -60,11 +59,7 @@ func (s *Suite[C]) RunTests(t *testing.T) { return } - if s.Cleanup != nil { - if err = s.Cleanup(*s.Config); err != nil { - t.Fatal(err) - } - } + s.cleanup(t) } type testFunction struct { @@ -158,13 +153,34 @@ func (s *Suite[C]) RunBenchmarks(b *testing.B) { return } - if err = s.Cleanup(*s.Config); err != nil { - b.Fatal(err) + s.cleanup(b) +} + +func (s *Suite[C]) createOrError() (restic.Backend, error) { + tr, err := backend.Transport(backend.TransportOptions{}) + if err != nil { + return nil, fmt.Errorf("cannot create transport for tests: %v", err) } + + be, err := s.Factory.Create(context.TODO(), s.Config, tr, nil) + if err != nil { + return nil, err + } + + _, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile}) + if err != nil && !be.IsNotExist(err) { + return nil, err + } + + if err == nil { + return nil, errors.New("config already exists") + } + + return be, nil } func (s *Suite[C]) create(t testing.TB) restic.Backend { - be, err := s.Create(*s.Config) + be, err := s.createOrError() if err != nil { t.Fatal(err) } @@ -172,13 +188,26 @@ func (s *Suite[C]) create(t testing.TB) restic.Backend { } func (s *Suite[C]) open(t testing.TB) restic.Backend { - be, err := s.Open(*s.Config) + tr, err := backend.Transport(backend.TransportOptions{}) + if err != nil { + t.Fatalf("cannot create transport for tests: %v", err) + } + + be, err := s.Factory.Open(context.TODO(), s.Config, tr, nil) if err != nil { t.Fatal(err) } return be } +func (s *Suite[C]) cleanup(t testing.TB) { + be := s.open(t) + if err := be.Delete(context.TODO()); err != nil { + t.Fatal(err) + } + s.close(t, be) +} + func (s *Suite[C]) close(t testing.TB, be restic.Backend) { err := be.Close() if err != nil { diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go index c4462495f..4a6a3f2a0 100644 --- a/internal/backend/test/tests.go +++ b/internal/backend/test/tests.go @@ -57,7 +57,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) { store(t, b, restic.ConfigFile, []byte("test config")) // now create the backend again, this must fail - _, err = s.Create(*s.Config) + _, err = s.createOrError() if err == nil { t.Fatalf("expected error not found for creating a backend with an existing config file") } From 705ad51bcc46cac89bcaf2b1b5f58d479a3e1e2e Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 8 Jun 2023 17:05:20 +0200 Subject: [PATCH 37/90] backend: check that StripPassword can be called --- internal/backend/test/tests.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go index 4a6a3f2a0..14faad0d5 100644 --- a/internal/backend/test/tests.go +++ b/internal/backend/test/tests.go @@ -36,6 +36,12 @@ func beTest(ctx context.Context, be restic.Backend, h restic.Handle) (bool, erro return err == nil, err } +// TestStripPasswordCall tests that the StripPassword method of a factory can be called without crashing. +// It does not verify whether passwords are removed correctly +func (s *Suite[C]) TestStripPasswordCall(t *testing.T) { + s.Factory.StripPassword("some random string") +} + // TestCreateWithConfig tests that creating a backend in a location which already // has a config file fails. func (s *Suite[C]) TestCreateWithConfig(t *testing.T) { From 50e0d5e6b5d0938e2926fcbed860c1af1e16fd0c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 8 Jun 2023 17:32:43 +0200 Subject: [PATCH 38/90] backend: Hardcode backend scheme in Factory Our ParseConfig implementations always expect a specific scheme, thus no other scheme would work. --- cmd/restic/global.go | 18 +++++++-------- internal/backend/azure/azure.go | 2 +- internal/backend/b2/b2.go | 2 +- internal/backend/gs/gs.go | 2 +- internal/backend/local/local.go | 2 +- .../backend/location/display_location_test.go | 4 ++-- internal/backend/location/location_test.go | 9 ++++---- internal/backend/location/registry.go | 22 ++++++++++++++----- internal/backend/mem/mem_backend.go | 1 + internal/backend/rclone/backend.go | 2 +- internal/backend/rest/rest.go | 2 +- internal/backend/s3/s3.go | 2 +- internal/backend/sftp/sftp.go | 2 +- internal/backend/swift/swift.go | 2 +- 14 files changed, 43 insertions(+), 29 deletions(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 1c13fb887..000ffac0b 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -100,15 +100,15 @@ var internalGlobalCtx context.Context func init() { backends := location.NewRegistry() - backends.Register("b2", b2.NewFactory()) - backends.Register("local", local.NewFactory()) - backends.Register("sftp", sftp.NewFactory()) - backends.Register("s3", s3.NewFactory()) - backends.Register("gs", gs.NewFactory()) - backends.Register("azure", azure.NewFactory()) - backends.Register("swift", swift.NewFactory()) - backends.Register("rest", rest.NewFactory()) - backends.Register("rclone", rclone.NewFactory()) + backends.Register(azure.NewFactory()) + backends.Register(b2.NewFactory()) + backends.Register(gs.NewFactory()) + backends.Register(local.NewFactory()) + backends.Register(rclone.NewFactory()) + backends.Register(rest.NewFactory()) + backends.Register(s3.NewFactory()) + backends.Register(sftp.NewFactory()) + backends.Register(swift.NewFactory()) globalOptions.backends = backends var cancel context.CancelFunc diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index b33b8dca6..a9267a945 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -45,7 +45,7 @@ const defaultListMaxItems = 5000 var _ restic.Backend = &Backend{} func NewFactory() location.Factory { - return location.NewHTTPBackendFactory(ParseConfig, location.NoPassword, Create, Open) + return location.NewHTTPBackendFactory("azure", ParseConfig, location.NoPassword, Create, Open) } func open(cfg Config, rt http.RoundTripper) (*Backend, error) { diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go index 700fff099..0bd3b994c 100644 --- a/internal/backend/b2/b2.go +++ b/internal/backend/b2/b2.go @@ -38,7 +38,7 @@ const defaultListMaxItems = 10 * 1000 var _ restic.Backend = &b2Backend{} func NewFactory() location.Factory { - return location.NewHTTPBackendFactory(ParseConfig, location.NoPassword, Create, Open) + return location.NewHTTPBackendFactory("b2", ParseConfig, location.NoPassword, Create, Open) } type sniffingRoundTripper struct { diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index 445ccc77d..5c12654d6 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -49,7 +49,7 @@ type Backend struct { var _ restic.Backend = &Backend{} func NewFactory() location.Factory { - return location.NewHTTPBackendFactory(ParseConfig, location.NoPassword, Create, Open) + return location.NewHTTPBackendFactory("gs", ParseConfig, location.NoPassword, Create, Open) } func getStorageClient(rt http.RoundTripper) (*storage.Client, error) { diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index e9d00abf7..4198102c2 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -31,7 +31,7 @@ type Local struct { var _ restic.Backend = &Local{} func NewFactory() location.Factory { - return location.NewLimitedBackendFactory(ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open)) + return location.NewLimitedBackendFactory("local", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open)) } const defaultLayout = "default" diff --git a/internal/backend/location/display_location_test.go b/internal/backend/location/display_location_test.go index 4a4055a84..19502d85b 100644 --- a/internal/backend/location/display_location_test.go +++ b/internal/backend/location/display_location_test.go @@ -10,8 +10,8 @@ import ( func TestStripPassword(t *testing.T) { registry := location.NewRegistry() - registry.Register("test", - location.NewHTTPBackendFactory[any, restic.Backend](nil, + registry.Register( + location.NewHTTPBackendFactory[any, restic.Backend]("test", nil, func(s string) string { return "cleaned" }, nil, nil, diff --git a/internal/backend/location/location_test.go b/internal/backend/location/location_test.go index 933f2fc08..b2623032e 100644 --- a/internal/backend/location/location_test.go +++ b/internal/backend/location/location_test.go @@ -14,6 +14,7 @@ type testConfig struct { func testFactory() location.Factory { return location.NewHTTPBackendFactory[testConfig, restic.Backend]( + "local", func(s string) (*testConfig, error) { return &testConfig{loc: s}, nil }, nil, nil, nil, @@ -22,12 +23,12 @@ func testFactory() location.Factory { func TestParse(t *testing.T) { registry := location.NewRegistry() - registry.Register("test", testFactory()) + registry.Register(testFactory()) - path := "test:example" + path := "local:example" u, err := location.Parse(registry, path) test.OK(t, err) - test.Equals(t, "test", u.Scheme) + test.Equals(t, "local", u.Scheme) test.Equals(t, &testConfig{loc: path}, u.Config) } @@ -43,7 +44,7 @@ func TestParseFallback(t *testing.T) { } registry := location.NewRegistry() - registry.Register("local", testFactory()) + registry.Register(testFactory()) for _, path := range fallbackTests { t.Run(path, func(t *testing.T) { diff --git a/internal/backend/location/registry.go b/internal/backend/location/registry.go index f15095590..a8818bd73 100644 --- a/internal/backend/location/registry.go +++ b/internal/backend/location/registry.go @@ -18,11 +18,11 @@ func NewRegistry() *Registry { } } -func (r *Registry) Register(scheme string, factory Factory) { - if r.factories[scheme] != nil { +func (r *Registry) Register(factory Factory) { + if r.factories[factory.Scheme()] != nil { panic("duplicate backend") } - r.factories[scheme] = factory + r.factories[factory.Scheme()] = factory } func (r *Registry) Lookup(scheme string) Factory { @@ -30,6 +30,7 @@ func (r *Registry) Lookup(scheme string) Factory { } type Factory interface { + Scheme() string ParseConfig(s string) (interface{}, error) StripPassword(s string) string Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) @@ -37,12 +38,17 @@ type Factory interface { } type genericBackendFactory[C any, T restic.Backend] struct { + scheme string parseConfigFn func(s string) (*C, error) stripPasswordFn func(s string) string createFn func(ctx context.Context, cfg C, rt http.RoundTripper, lim limiter.Limiter) (T, error) openFn func(ctx context.Context, cfg C, rt http.RoundTripper, lim limiter.Limiter) (T, error) } +func (f *genericBackendFactory[C, T]) Scheme() string { + return f.scheme +} + func (f *genericBackendFactory[C, T]) ParseConfig(s string) (interface{}, error) { return f.parseConfigFn(s) } @@ -59,12 +65,15 @@ func (f *genericBackendFactory[C, T]) Open(ctx context.Context, cfg interface{}, return f.openFn(ctx, *cfg.(*C), rt, lim) } -func NewHTTPBackendFactory[C any, T restic.Backend](parseConfigFn func(s string) (*C, error), +func NewHTTPBackendFactory[C any, T restic.Backend]( + scheme string, + parseConfigFn func(s string) (*C, error), stripPasswordFn func(s string) string, createFn func(ctx context.Context, cfg C, rt http.RoundTripper) (T, error), openFn func(ctx context.Context, cfg C, rt http.RoundTripper) (T, error)) Factory { return &genericBackendFactory[C, T]{ + scheme: scheme, parseConfigFn: parseConfigFn, stripPasswordFn: stripPasswordFn, createFn: func(ctx context.Context, cfg C, rt http.RoundTripper, _ limiter.Limiter) (T, error) { @@ -76,12 +85,15 @@ func NewHTTPBackendFactory[C any, T restic.Backend](parseConfigFn func(s string) } } -func NewLimitedBackendFactory[C any, T restic.Backend](parseConfigFn func(s string) (*C, error), +func NewLimitedBackendFactory[C any, T restic.Backend]( + scheme string, + parseConfigFn func(s string) (*C, error), stripPasswordFn func(s string) string, createFn func(ctx context.Context, cfg C, lim limiter.Limiter) (T, error), openFn func(ctx context.Context, cfg C, lim limiter.Limiter) (T, error)) Factory { return &genericBackendFactory[C, T]{ + scheme: scheme, parseConfigFn: parseConfigFn, stripPasswordFn: stripPasswordFn, createFn: func(ctx context.Context, cfg C, _ http.RoundTripper, lim limiter.Limiter) (T, error) { diff --git a/internal/backend/mem/mem_backend.go b/internal/backend/mem/mem_backend.go index a467d33f7..86ec48756 100644 --- a/internal/backend/mem/mem_backend.go +++ b/internal/backend/mem/mem_backend.go @@ -27,6 +27,7 @@ func NewFactory() location.Factory { be := New() return location.NewHTTPBackendFactory[struct{}, *MemoryBackend]( + "mem", func(s string) (*struct{}, error) { return &struct{}{}, nil }, diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go index f3a97ef75..fd6f5b262 100644 --- a/internal/backend/rclone/backend.go +++ b/internal/backend/rclone/backend.go @@ -38,7 +38,7 @@ type Backend struct { } func NewFactory() location.Factory { - return location.NewLimitedBackendFactory(ParseConfig, location.NoPassword, Create, Open) + return location.NewLimitedBackendFactory("rclone", ParseConfig, location.NoPassword, Create, Open) } // run starts command with args and initializes the StdioConn. diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index 4fb2d54de..8391df681 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -31,7 +31,7 @@ type Backend struct { } func NewFactory() location.Factory { - return location.NewHTTPBackendFactory(ParseConfig, StripPassword, Create, Open) + return location.NewHTTPBackendFactory("rest", ParseConfig, StripPassword, Create, Open) } // the REST API protocol version is decided by HTTP request headers, these are the constants. diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index 10512e809..3fe32d215 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -33,7 +33,7 @@ type Backend struct { var _ restic.Backend = &Backend{} func NewFactory() location.Factory { - return location.NewHTTPBackendFactory(ParseConfig, location.NoPassword, Create, Open) + return location.NewHTTPBackendFactory("s3", ParseConfig, location.NoPassword, Create, Open) } const defaultLayout = "default" diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index 1e12df808..3e127ef05 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -44,7 +44,7 @@ type SFTP struct { var _ restic.Backend = &SFTP{} func NewFactory() location.Factory { - return location.NewLimitedBackendFactory(ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open)) + return location.NewLimitedBackendFactory("sftp", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open)) } const defaultLayout = "default" diff --git a/internal/backend/swift/swift.go b/internal/backend/swift/swift.go index 019456be7..1cfc0a65b 100644 --- a/internal/backend/swift/swift.go +++ b/internal/backend/swift/swift.go @@ -36,7 +36,7 @@ type beSwift struct { var _ restic.Backend = &beSwift{} func NewFactory() location.Factory { - return location.NewHTTPBackendFactory(ParseConfig, location.NoPassword, Open, Open) + return location.NewHTTPBackendFactory("swift", ParseConfig, location.NoPassword, Open, Open) } // Open opens the swift backend at a container in region. The container is From b5511e8e4c4231bb4b746ec727c963b527fe896f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 8 Jun 2023 19:35:20 +0200 Subject: [PATCH 39/90] Fix linter warnings --- internal/backend/azure/azure_test.go | 6 ++-- internal/backend/b2/b2_test.go | 6 ++-- internal/backend/gs/gs_test.go | 6 ++-- internal/backend/rest/rest_test.go | 11 +++--- internal/backend/s3/s3_test.go | 52 +++++++++++++--------------- internal/backend/test/tests.go | 2 +- 6 files changed, 38 insertions(+), 45 deletions(-) diff --git a/internal/backend/azure/azure_test.go b/internal/backend/azure/azure_test.go index 8465cc3b0..33f65bd52 100644 --- a/internal/backend/azure/azure_test.go +++ b/internal/backend/azure/azure_test.go @@ -17,7 +17,7 @@ import ( rtest "github.com/restic/restic/internal/test" ) -func newAzureTestSuite(t testing.TB) *test.Suite[azure.Config] { +func newAzureTestSuite() *test.Suite[azure.Config] { return &test.Suite[azure.Config]{ // do not use excessive data MinimalData: true, @@ -59,7 +59,7 @@ func TestBackendAzure(t *testing.T) { } t.Logf("run tests") - newAzureTestSuite(t).RunTests(t) + newAzureTestSuite().RunTests(t) } func BenchmarkBackendAzure(t *testing.B) { @@ -77,7 +77,7 @@ func BenchmarkBackendAzure(t *testing.B) { } t.Logf("run tests") - newAzureTestSuite(t).RunBenchmarks(t) + newAzureTestSuite().RunBenchmarks(t) } func TestUploadLargeFile(t *testing.T) { diff --git a/internal/backend/b2/b2_test.go b/internal/backend/b2/b2_test.go index 348af9095..ab1dcd37b 100644 --- a/internal/backend/b2/b2_test.go +++ b/internal/backend/b2/b2_test.go @@ -12,7 +12,7 @@ import ( rtest "github.com/restic/restic/internal/test" ) -func newB2TestSuite(t testing.TB) *test.Suite[b2.Config] { +func newB2TestSuite() *test.Suite[b2.Config] { return &test.Suite[b2.Config]{ // do not use excessive data MinimalData: true, @@ -59,10 +59,10 @@ func TestBackendB2(t *testing.T) { }() testVars(t) - newB2TestSuite(t).RunTests(t) + newB2TestSuite().RunTests(t) } func BenchmarkBackendb2(t *testing.B) { testVars(t) - newB2TestSuite(t).RunBenchmarks(t) + newB2TestSuite().RunBenchmarks(t) } diff --git a/internal/backend/gs/gs_test.go b/internal/backend/gs/gs_test.go index 47085dc4e..22953cad3 100644 --- a/internal/backend/gs/gs_test.go +++ b/internal/backend/gs/gs_test.go @@ -11,7 +11,7 @@ import ( rtest "github.com/restic/restic/internal/test" ) -func newGSTestSuite(t testing.TB) *test.Suite[gs.Config] { +func newGSTestSuite() *test.Suite[gs.Config] { return &test.Suite[gs.Config]{ // do not use excessive data MinimalData: true, @@ -56,7 +56,7 @@ func TestBackendGS(t *testing.T) { } t.Logf("run tests") - newGSTestSuite(t).RunTests(t) + newGSTestSuite().RunTests(t) } func BenchmarkBackendGS(t *testing.B) { @@ -77,5 +77,5 @@ func BenchmarkBackendGS(t *testing.B) { } t.Logf("run tests") - newGSTestSuite(t).RunBenchmarks(t) + newGSTestSuite().RunBenchmarks(t) } diff --git a/internal/backend/rest/rest_test.go b/internal/backend/rest/rest_test.go index 60cc40afe..6a5b4f8a5 100644 --- a/internal/backend/rest/rest_test.go +++ b/internal/backend/rest/rest_test.go @@ -65,7 +65,7 @@ func runRESTServer(ctx context.Context, t testing.TB, dir string) (*url.URL, fun return url, cleanup } -func newTestSuite(_ context.Context, t testing.TB, url *url.URL, minimalData bool) *test.Suite[rest.Config] { +func newTestSuite(url *url.URL, minimalData bool) *test.Suite[rest.Config] { return &test.Suite[rest.Config]{ MinimalData: minimalData, @@ -94,7 +94,7 @@ func TestBackendREST(t *testing.T) { serverURL, cleanup := runRESTServer(ctx, t, dir) defer cleanup() - newTestSuite(ctx, t, serverURL, false).RunTests(t) + newTestSuite(serverURL, false).RunTests(t) } func TestBackendRESTExternalServer(t *testing.T) { @@ -108,10 +108,7 @@ func TestBackendRESTExternalServer(t *testing.T) { t.Fatal(err) } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - newTestSuite(ctx, t, cfg.URL, true).RunTests(t) + newTestSuite(cfg.URL, true).RunTests(t) } func BenchmarkBackendREST(t *testing.B) { @@ -122,5 +119,5 @@ func BenchmarkBackendREST(t *testing.B) { serverURL, cleanup := runRESTServer(ctx, t, dir) defer cleanup() - newTestSuite(ctx, t, serverURL, false).RunBenchmarks(t) + newTestSuite(serverURL, false).RunBenchmarks(t) } diff --git a/internal/backend/s3/s3_test.go b/internal/backend/s3/s3_test.go index e645fe03e..17f5f7016 100644 --- a/internal/backend/s3/s3_test.go +++ b/internal/backend/s3/s3_test.go @@ -94,35 +94,31 @@ func newRandomCredentials(t testing.TB) (key, secret string) { return key, secret } -func newMinioTestSuite(ctx context.Context, t testing.TB, key string, secret string) *test.Suite[s3.Config] { - return &test.Suite[s3.Config]{ - // NewConfig returns a config for a new temporary backend that will be used in tests. - NewConfig: func() (*s3.Config, error) { - cfg := s3.NewConfig() - cfg.Endpoint = "localhost:9000" - cfg.Bucket = "restictestbucket" - cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) - cfg.UseHTTP = true - cfg.KeyID = key - cfg.Secret = options.NewSecretString(secret) - return &cfg, nil - }, - - Factory: s3.NewFactory(), - } -} - -func createMinioTestSuite(t testing.TB) (*test.Suite[s3.Config], func()) { +func newMinioTestSuite(t testing.TB) (*test.Suite[s3.Config], func()) { ctx, cancel := context.WithCancel(context.Background()) tempdir := rtest.TempDir(t) key, secret := newRandomCredentials(t) cleanup := runMinio(ctx, t, tempdir, key, secret) - return newMinioTestSuite(ctx, t, key, secret), func() { - defer cancel() - defer cleanup() - } + return &test.Suite[s3.Config]{ + // NewConfig returns a config for a new temporary backend that will be used in tests. + NewConfig: func() (*s3.Config, error) { + cfg := s3.NewConfig() + cfg.Endpoint = "localhost:9000" + cfg.Bucket = "restictestbucket" + cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) + cfg.UseHTTP = true + cfg.KeyID = key + cfg.Secret = options.NewSecretString(secret) + return &cfg, nil + }, + + Factory: s3.NewFactory(), + }, func() { + defer cancel() + defer cleanup() + } } func TestBackendMinio(t *testing.T) { @@ -139,7 +135,7 @@ func TestBackendMinio(t *testing.T) { return } - suite, cleanup := createMinioTestSuite(t) + suite, cleanup := newMinioTestSuite(t) defer cleanup() suite.RunTests(t) @@ -153,13 +149,13 @@ func BenchmarkBackendMinio(t *testing.B) { return } - suite, cleanup := createMinioTestSuite(t) + suite, cleanup := newMinioTestSuite(t) defer cleanup() suite.RunBenchmarks(t) } -func newS3TestSuite(t testing.TB) *test.Suite[s3.Config] { +func newS3TestSuite() *test.Suite[s3.Config] { return &test.Suite[s3.Config]{ // do not use excessive data MinimalData: true, @@ -202,7 +198,7 @@ func TestBackendS3(t *testing.T) { } t.Logf("run tests") - newS3TestSuite(t).RunTests(t) + newS3TestSuite().RunTests(t) } func BenchmarkBackendS3(t *testing.B) { @@ -220,5 +216,5 @@ func BenchmarkBackendS3(t *testing.B) { } t.Logf("run tests") - newS3TestSuite(t).RunBenchmarks(t) + newS3TestSuite().RunBenchmarks(t) } diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go index 14faad0d5..c2e5d0fc0 100644 --- a/internal/backend/test/tests.go +++ b/internal/backend/test/tests.go @@ -38,7 +38,7 @@ func beTest(ctx context.Context, be restic.Backend, h restic.Handle) (bool, erro // TestStripPasswordCall tests that the StripPassword method of a factory can be called without crashing. // It does not verify whether passwords are removed correctly -func (s *Suite[C]) TestStripPasswordCall(t *testing.T) { +func (s *Suite[C]) TestStripPasswordCall(_ *testing.T) { s.Factory.StripPassword("some random string") } From cbf87fbdb35442c6b67cb333072c1605896e8bf9 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 17 Jun 2023 14:40:10 +0200 Subject: [PATCH 40/90] init: don't include password in debug log --- cmd/restic/global.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 000ffac0b..823a82e36 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -623,7 +623,7 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio // Create the backend specified by URI. func create(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) { - debug.Log("parsing location %v", s) + debug.Log("parsing location %v", location.StripPassword(gopts.backends, s)) loc, err := location.Parse(gopts.backends, s) if err != nil { return nil, err From 182b9796e4135b756d90e7c68f2faaad89fa6ebe Mon Sep 17 00:00:00 2001 From: Gautam Menghani Date: Fri, 16 Jun 2023 23:13:41 +0530 Subject: [PATCH 41/90] Issue #3624: Preserve oldest snapshot when keep-* values are not satisfied --- changelog/unreleased/issue-3624 | 12 ++++++++++++ internal/restic/snapshot_policy.go | 2 +- .../restic/testdata/policy_keep_snapshots_16 | 18 ++++++++++++++++++ .../restic/testdata/policy_keep_snapshots_17 | 18 ++++++++++++++++++ .../restic/testdata/policy_keep_snapshots_39 | 17 +++++++++++++++++ 5 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/issue-3624 diff --git a/changelog/unreleased/issue-3624 b/changelog/unreleased/issue-3624 new file mode 100644 index 000000000..53837e61c --- /dev/null +++ b/changelog/unreleased/issue-3624 @@ -0,0 +1,12 @@ +Enhancement: Keep oldest snapshot when there aren't enough snapshots + +The `forget` command does not preserve the oldest snapshot incase the +keep-* parameters are not satisfied, which led to users not being able to +preserve old data. Now, restic will always preserve the oldest snapshot +whenever any of the keep-* options to the `forget` command are not +satisfied. + + +https://github.com/restic/restic/issues/3624 +https://github.com/restic/restic/pull/4366 +https://forum.restic.net/t/keeping-yearly-snapshots-policy-when-backup-began-during-the-year/4670/2 diff --git a/internal/restic/snapshot_policy.go b/internal/restic/snapshot_policy.go index bec594707..1d6e129a2 100644 --- a/internal/restic/snapshot_policy.go +++ b/internal/restic/snapshot_policy.go @@ -256,7 +256,7 @@ func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots, reason // -1 means "keep all" if b.Count > 0 || b.Count == -1 { val := b.bucker(cur.Time, nr) - if val != b.Last { + if val != b.Last || nr == len(list)-1 { debug.Log("keep %v %v, bucker %v, val %v\n", cur.Time, cur.id.Str(), i, val) keepSnap = true buckets[i].Last = val diff --git a/internal/restic/testdata/policy_keep_snapshots_16 b/internal/restic/testdata/policy_keep_snapshots_16 index d0cae94b5..da6f43a1c 100644 --- a/internal/restic/testdata/policy_keep_snapshots_16 +++ b/internal/restic/testdata/policy_keep_snapshots_16 @@ -14,6 +14,11 @@ "time": "2014-11-22T10:20:30Z", "tree": null, "paths": null + }, + { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null } ], "reasons": [ @@ -55,6 +60,19 @@ "counters": { "yearly": 7 } + }, + { + "snapshot": { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "yearly snapshot" + ], + "counters": { + "yearly": 6 + } } ] } \ No newline at end of file diff --git a/internal/restic/testdata/policy_keep_snapshots_17 b/internal/restic/testdata/policy_keep_snapshots_17 index 742b8005b..ee728d4e0 100644 --- a/internal/restic/testdata/policy_keep_snapshots_17 +++ b/internal/restic/testdata/policy_keep_snapshots_17 @@ -49,6 +49,11 @@ "time": "2014-11-22T10:20:30Z", "tree": null, "paths": null + }, + { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null } ], "reasons": [ @@ -201,6 +206,19 @@ "counters": { "yearly": 7 } + }, + { + "snapshot": { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "yearly snapshot" + ], + "counters": { + "yearly": 6 + } } ] } \ No newline at end of file diff --git a/internal/restic/testdata/policy_keep_snapshots_39 b/internal/restic/testdata/policy_keep_snapshots_39 index a8e6ca827..4b111503b 100644 --- a/internal/restic/testdata/policy_keep_snapshots_39 +++ b/internal/restic/testdata/policy_keep_snapshots_39 @@ -57,6 +57,11 @@ "time": "2014-08-22T10:20:30Z", "tree": null, "paths": null + }, + { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null } ], "reasons": [ @@ -189,6 +194,18 @@ "monthly snapshot" ], "counters": {"Monthly": -1, "Yearly": -1} + }, + { + "snapshot": { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "monthly snapshot", + "yearly snapshot" + ], + "counters": {"Monthly": -1, "Yearly": -1} } ] } \ No newline at end of file From 1257c2c075ea866cb589fa3ec570a6f8bd11a14d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 16 Jun 2023 21:28:49 +0200 Subject: [PATCH 42/90] forget: Add comments to snapshot policy --- internal/restic/snapshot_policy.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/restic/snapshot_policy.go b/internal/restic/snapshot_policy.go index 1d6e129a2..22939bd6c 100644 --- a/internal/restic/snapshot_policy.go +++ b/internal/restic/snapshot_policy.go @@ -183,6 +183,7 @@ type KeepReason struct { // according to the policy p. list is sorted in the process. reasons contains // the reasons to keep each snapshot, it is in the same order as keep. func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots, reasons []KeepReason) { + // sort newest snapshots first sort.Stable(list) if p.Empty() { @@ -256,6 +257,8 @@ func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots, reason // -1 means "keep all" if b.Count > 0 || b.Count == -1 { val := b.bucker(cur.Time, nr) + // also keep the oldest snapshot if the bucket has some counts left. This maximizes the + // the history length kept while some counts are left. if val != b.Last || nr == len(list)-1 { debug.Log("keep %v %v, bucker %v, val %v\n", cur.Time, cur.id.Str(), i, val) keepSnap = true From 3888c21a27d807cba58f1dcc56e455b3adab40c9 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 16 Jun 2023 21:35:20 +0200 Subject: [PATCH 43/90] reword changelog --- changelog/unreleased/issue-3624 | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/changelog/unreleased/issue-3624 b/changelog/unreleased/issue-3624 index 53837e61c..ce3fe57aa 100644 --- a/changelog/unreleased/issue-3624 +++ b/changelog/unreleased/issue-3624 @@ -1,11 +1,8 @@ -Enhancement: Keep oldest snapshot when there aren't enough snapshots - -The `forget` command does not preserve the oldest snapshot incase the -keep-* parameters are not satisfied, which led to users not being able to -preserve old data. Now, restic will always preserve the oldest snapshot -whenever any of the keep-* options to the `forget` command are not -satisfied. +Enhancement: Keep oldest snapshot when there are not enough snapshots +The `forget` command now additionally preserves the oldest snapshot if fewer +snapshots are kept than allowed by the `--keep-*` parameters. This maximizes +amount of history kept while the specified limits are not yet reached. https://github.com/restic/restic/issues/3624 https://github.com/restic/restic/pull/4366 From 8da5a6649bd2016a164b6bb471bafaf7fe2d3d1a Mon Sep 17 00:00:00 2001 From: Gautam Menghani Date: Sat, 17 Jun 2023 18:04:35 +0530 Subject: [PATCH 44/90] Preserve oldest snapshot when keep-within* does not collect enough --- internal/restic/snapshot_policy.go | 2 +- .../restic/testdata/policy_keep_snapshots_35 | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/internal/restic/snapshot_policy.go b/internal/restic/snapshot_policy.go index 22939bd6c..0ff0c5ec8 100644 --- a/internal/restic/snapshot_policy.go +++ b/internal/restic/snapshot_policy.go @@ -278,7 +278,7 @@ func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots, reason if cur.Time.After(t) { val := b.bucker(cur.Time, nr) - if val != b.Last { + if val != b.Last || nr == len(list)-1 { debug.Log("keep %v, time %v, ID %v, bucker %v, val %v %v\n", b.reason, cur.Time, cur.id.Str(), i, val, b.Last) keepSnap = true bucketsWithin[i].Last = val diff --git a/internal/restic/testdata/policy_keep_snapshots_35 b/internal/restic/testdata/policy_keep_snapshots_35 index a4def907a..ece4ddbd2 100644 --- a/internal/restic/testdata/policy_keep_snapshots_35 +++ b/internal/restic/testdata/policy_keep_snapshots_35 @@ -44,6 +44,11 @@ "time": "2014-11-22T10:20:30Z", "tree": null, "paths": null + }, + { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null } ], "reasons": [ @@ -152,6 +157,17 @@ "yearly within 9999y" ], "counters": {} + }, + { + "snapshot": { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + "matches": [ + "yearly within 9999y" + ], + "counters": {} } ] } \ No newline at end of file From 93038ed8f4c73cdebb54d8d3af7ea34b8bb01c2b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 17 Jun 2023 15:25:08 +0200 Subject: [PATCH 45/90] s3: restore retries for minio tests --- internal/backend/s3/s3_test.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/internal/backend/s3/s3_test.go b/internal/backend/s3/s3_test.go index 17f5f7016..3051d8ddb 100644 --- a/internal/backend/s3/s3_test.go +++ b/internal/backend/s3/s3_test.go @@ -7,15 +7,18 @@ import ( "fmt" "io" "net" + "net/http" "os" "os/exec" "path/filepath" "testing" "time" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/backend/test" "github.com/restic/restic/internal/options" + "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -114,7 +117,18 @@ func newMinioTestSuite(t testing.TB) (*test.Suite[s3.Config], func()) { return &cfg, nil }, - Factory: s3.NewFactory(), + Factory: location.NewHTTPBackendFactory("s3", s3.ParseConfig, location.NoPassword, func(ctx context.Context, cfg s3.Config, rt http.RoundTripper) (be restic.Backend, err error) { + for i := 0; i < 10; i++ { + be, err = s3.Create(ctx, cfg, rt) + if err != nil { + t.Logf("s3 open: try %d: error %v", i, err) + time.Sleep(500 * time.Millisecond) + continue + } + break + } + return be, err + }, s3.Open), }, func() { defer cancel() defer cleanup() From 8dcb0c4a9d830955888aa928fb59533a9fbf71d7 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 17 Jun 2023 15:45:54 +0200 Subject: [PATCH 46/90] doc: improve description of caching behavior of the check command --- doc/045_working_with_repos.rst | 10 ++++++++++ doc/manual_rest.rst | 7 ++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index 00d87a450..82a20bac4 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -232,6 +232,8 @@ modifying the repository. Instead restic will only print the actions it would perform. +.. _checking-integrity: + Checking integrity and consistency ================================== @@ -284,6 +286,14 @@ If the repository structure is intact, restic will show that no errors were foun check snapshots, trees and blobs no errors were found +By default, check creates a new temporary cache directory to verify that the +data stored in the repository is intact. To reuse the existing cache, you can +use the ``--with-cache`` flag. + +If the cache directory is not explicitly set, then ``check`` creates its +temporary cache directory in the temporary directory, see :ref:`temporary_files`. +Otherwise, the specified cache directory is used, as described in :ref:`caching`. + By default, the ``check`` command does not verify that the actual pack files on disk in the repository are unmodified, because doing so requires reading a copy of every pack file in the repository. To tell restic to also verify the diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index b83a152bf..71f5e192b 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -418,8 +418,6 @@ instead of the default, set the environment variable like this: $ restic -r /srv/restic-repo backup ~/work -If the environment variable ``$RESTIC_CACHE_DIR`` is not set, ``check`` creates its temporary cache directory in the temporary directory. If ``$RESTIC_CACHE_DIR`` is set, ``check`` creates its temporary cache directory in ``$RESTIC_CACHE_DIR``. - .. _caching: Caching @@ -443,6 +441,10 @@ The command line parameter ``--cache-dir`` or the environment variable parameter ``--no-cache`` disables the cache entirely. In this case, all data is loaded from the repository. +If a cache location is explicitly specified, then the ``check`` command will use +that location to store its temporary cache. See :ref:`checking-integrity` for +more details. + The cache is ephemeral: When a file cannot be read from the cache, it is loaded from the repository. @@ -452,4 +454,3 @@ time it is used, so by looking at the timestamps of the sub directories of the cache directory it can decide which sub directories are old and probably not needed any more. You can either remove these directories manually, or run a restic command with the ``--cleanup-cache`` flag. - From 8c02ebb0299cf517aa7a48bfc27bdf1ad5a5b7ba Mon Sep 17 00:00:00 2001 From: Andrew Gunnerson Date: Mon, 19 Jun 2023 14:24:47 -0400 Subject: [PATCH 47/90] Add support for extended attributes on symlinks Linux allows the use of non-`user.` extended attributes on symlinks. One of the main users of this functionality is SELinux's `security.selinux` xattr for storing a path's label. By storing symlink xattrs, restic is now suitable for backing up the root filesystem on Linux distributions that use SELinux. This commit adds support for symlink xattrs when backing up data, restoring data, and mounting snapshots via a fuse mount. All calls to the xattr library have been updated to the use `L` variants of the various functions, which always operate on the path given, without following symlinks. Fixes: #4375 Signed-off-by: Andrew Gunnerson --- changelog/unreleased/issue-4375 | 8 ++++++++ internal/fuse/link.go | 20 ++++++++++++++++++++ internal/restic/node.go | 4 ---- internal/restic/node_xattr.go | 6 +++--- 4 files changed, 31 insertions(+), 7 deletions(-) create mode 100644 changelog/unreleased/issue-4375 diff --git a/changelog/unreleased/issue-4375 b/changelog/unreleased/issue-4375 new file mode 100644 index 000000000..6ce68c2ba --- /dev/null +++ b/changelog/unreleased/issue-4375 @@ -0,0 +1,8 @@ +Enhancement: Add support for extended attributes on symlinks + +Restic now supports extended attributes on symlinks when backing up, +restoring, or FUSE-mounting snapshots. This includes, for example, the +`security.selinux` xattr on Linux distributions that use SELinux. + +https://github.com/restic/restic/issues/4375 +https://github.com/restic/restic/pull/4379 diff --git a/internal/fuse/link.go b/internal/fuse/link.go index c89451602..43b00a855 100644 --- a/internal/fuse/link.go +++ b/internal/fuse/link.go @@ -6,6 +6,8 @@ package fuse import ( "context" + "github.com/restic/restic/internal/debug" + "github.com/anacrolix/fuse" "github.com/anacrolix/fuse/fs" "github.com/restic/restic/internal/restic" @@ -46,3 +48,21 @@ func (l *link) Attr(_ context.Context, a *fuse.Attr) error { return nil } + +func (l *link) Listxattr(_ context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error { + debug.Log("Listxattr(%v, %v)", l.node.Name, req.Size) + for _, attr := range l.node.ExtendedAttributes { + resp.Append(attr.Name) + } + return nil +} + +func (l *link) Getxattr(_ context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { + debug.Log("Getxattr(%v, %v, %v)", l.node.Name, req.Name, req.Size) + attrval := l.node.GetExtendedAttribute(req.Name) + if attrval != nil { + resp.Xattr = attrval + return nil + } + return fuse.ErrNoXattr +} diff --git a/internal/restic/node.go b/internal/restic/node.go index 7d2a1434e..f2d9f2315 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -609,10 +609,6 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error { } func (node *Node) fillExtendedAttributes(path string) error { - if node.Type == "symlink" { - return nil - } - xattrs, err := Listxattr(path) debug.Log("fillExtendedAttributes(%v) %v %v", path, xattrs, err) if err != nil { diff --git a/internal/restic/node_xattr.go b/internal/restic/node_xattr.go index a2eed39c0..ea9eafe94 100644 --- a/internal/restic/node_xattr.go +++ b/internal/restic/node_xattr.go @@ -13,20 +13,20 @@ import ( // Getxattr retrieves extended attribute data associated with path. func Getxattr(path, name string) ([]byte, error) { - b, err := xattr.Get(path, name) + b, err := xattr.LGet(path, name) return b, handleXattrErr(err) } // Listxattr retrieves a list of names of extended attributes associated with the // given path in the file system. func Listxattr(path string) ([]string, error) { - l, err := xattr.List(path) + l, err := xattr.LList(path) return l, handleXattrErr(err) } // Setxattr associates name and data together as an attribute of path. func Setxattr(path, name string, data []byte) error { - return handleXattrErr(xattr.Set(path, name, data)) + return handleXattrErr(xattr.LSet(path, name, data)) } func handleXattrErr(err error) error { From 70fb5548544459c2b09bdcf9d920f549b359fd69 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Jul 2023 01:23:05 +0000 Subject: [PATCH 48/90] build(deps): bump golang.org/x/sys from 0.8.0 to 0.9.0 Bumps [golang.org/x/sys](https://github.com/golang/sys) from 0.8.0 to 0.9.0. - [Commits](https://github.com/golang/sys/compare/v0.8.0...v0.9.0) --- updated-dependencies: - dependency-name: golang.org/x/sys dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 08e2b9846..c38890085 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( golang.org/x/net v0.10.0 golang.org/x/oauth2 v0.8.0 golang.org/x/sync v0.2.0 - golang.org/x/sys v0.8.0 + golang.org/x/sys v0.9.0 golang.org/x/term v0.8.0 golang.org/x/text v0.9.0 google.golang.org/api v0.116.0 diff --git a/go.sum b/go.sum index 1e40934bf..c3b4057ca 100644 --- a/go.sum +++ b/go.sum @@ -212,8 +212,8 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= From f08ba1a005ac9c8797a970a16aff6f3141ac1991 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Jul 2023 01:23:19 +0000 Subject: [PATCH 49/90] build(deps): bump github.com/Azure/azure-sdk-for-go/sdk/azcore Bumps [github.com/Azure/azure-sdk-for-go/sdk/azcore](https://github.com/Azure/azure-sdk-for-go) from 1.6.0 to 1.6.1. - [Release notes](https://github.com/Azure/azure-sdk-for-go/releases) - [Changelog](https://github.com/Azure/azure-sdk-for-go/blob/main/documentation/release.md) - [Commits](https://github.com/Azure/azure-sdk-for-go/compare/sdk/azcore/v1.6.0...sdk/azcore/v1.6.1) --- updated-dependencies: - dependency-name: github.com/Azure/azure-sdk-for-go/sdk/azcore dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 08e2b9846..2b8cbcede 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/restic/restic require ( cloud.google.com/go/storage v1.30.1 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 github.com/anacrolix/fuse v0.2.0 github.com/cenkalti/backoff/v4 v4.2.0 diff --git a/go.sum b/go.sum index 1e40934bf..2b827b0a9 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCta cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 h1:SEy2xmstIphdPwNBUi7uhvjyjhVKISfwjfOJmuy7kg4= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= From d2f8f9de238c3ffba1840eee39faeae5c04bea9d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Jul 2023 15:32:55 +0000 Subject: [PATCH 50/90] build(deps): bump golang.org/x/oauth2 from 0.8.0 to 0.9.0 Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from 0.8.0 to 0.9.0. - [Commits](https://github.com/golang/oauth2/compare/v0.8.0...v0.9.0) --- updated-dependencies: - dependency-name: golang.org/x/oauth2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index c38890085..cbccae944 100644 --- a/go.mod +++ b/go.mod @@ -24,13 +24,13 @@ require ( github.com/restic/chunker v0.4.0 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 - golang.org/x/crypto v0.9.0 - golang.org/x/net v0.10.0 - golang.org/x/oauth2 v0.8.0 + golang.org/x/crypto v0.10.0 + golang.org/x/net v0.11.0 + golang.org/x/oauth2 v0.9.0 golang.org/x/sync v0.2.0 golang.org/x/sys v0.9.0 - golang.org/x/term v0.8.0 - golang.org/x/text v0.9.0 + golang.org/x/term v0.9.0 + golang.org/x/text v0.10.0 google.golang.org/api v0.116.0 ) diff --git a/go.sum b/go.sum index c3b4057ca..8eb28d703 100644 --- a/go.sum +++ b/go.sum @@ -170,8 +170,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -187,11 +187,11 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= -golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs= +golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -215,14 +215,14 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= From 6e7c6674ad2e23797e4c77f1276ad63c6aef0855 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Jul 2023 16:11:44 +0000 Subject: [PATCH 51/90] build(deps): bump google.golang.org/api from 0.116.0 to 0.129.0 Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.116.0 to 0.129.0. - [Release notes](https://github.com/googleapis/google-api-go-client/releases) - [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.116.0...v0.129.0) --- updated-dependencies: - dependency-name: google.golang.org/api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 19 ++++++++------ go.sum | 82 ++++++++++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 76 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index d70773100..1554eca40 100644 --- a/go.mod +++ b/go.mod @@ -27,16 +27,16 @@ require ( golang.org/x/crypto v0.10.0 golang.org/x/net v0.11.0 golang.org/x/oauth2 v0.9.0 - golang.org/x/sync v0.2.0 + golang.org/x/sync v0.3.0 golang.org/x/sys v0.9.0 golang.org/x/term v0.9.0 golang.org/x/text v0.10.0 - google.golang.org/api v0.116.0 + google.golang.org/api v0.129.0 ) require ( cloud.google.com/go v0.110.0 // indirect - cloud.google.com/go/compute v1.19.0 // indirect + cloud.google.com/go/compute v1.19.3 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v0.13.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect @@ -47,9 +47,10 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b // indirect + github.com/google/s2a-go v0.1.4 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.8.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect + github.com/googleapis/gax-go/v2 v2.11.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect @@ -63,9 +64,11 @@ require ( go.opencensus.io v0.24.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 // indirect - google.golang.org/grpc v1.54.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/grpc v1.56.1 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index a074debdf..7893b5e39 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,13 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= -cloud.google.com/go/compute v1.19.0 h1:+9zda3WGgW1ZSTlVppLCYFIr48Pa35q1uG2N1itbCEQ= -cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= +cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds= +cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= -cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 h1:SEy2xmstIphdPwNBUi7uhvjyjhVKISfwjfOJmuy7kg4= @@ -22,9 +22,11 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/anacrolix/fuse v0.2.0 h1:pc+To78kI2d/WUjIyrsdqeJQAesuwpGxlI3h1nAv3Do= github.com/anacrolix/fuse v0.2.0/go.mod h1:Kfu02xBwnySDpH3N23BmrP3MDfwAQGRLUCj6XyeOvBQ= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -32,6 +34,11 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -48,9 +55,12 @@ github.com/elithrar/simple-scrypt v1.3.0/go.mod h1:U2XQRI95XHY0St410VE3UjT7vuKb1 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= @@ -62,14 +72,17 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -86,13 +99,16 @@ github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdf github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b h1:8htHrh2bw9c7Idkb7YNac+ZpTqLMjRpI+FWu51ltaQc= github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= +github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= -github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= -github.com/googleapis/gax-go/v2 v2.8.0 h1:UBtEZqx1bjXtOQ5BVTkuYghXrr3N4V123VKJK67vJZc= -github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= +github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= +github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= +github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= @@ -141,6 +157,7 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/restic/chunker v0.4.0 h1:YUPYCUn70MYP7VO4yllypp2SjmsRhRJaad3xKu1QFRw= github.com/restic/chunker v0.4.0/go.mod h1:z0cH2BejpW636LXw0R/BGyv+Ey8+m9QGiOanDHItzyw= github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -156,6 +173,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -164,12 +182,16 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -177,32 +199,41 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs= golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -210,17 +241,22 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -230,29 +266,39 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/api v0.116.0 h1:09tOPVufPwfm5W4aA8EizGHJ7BcoRDsIareM2a15gO4= -google.golang.org/api v0.116.0/go.mod h1:9cD4/t6uvd9naoEJFA+M96d0IuB6BqFuyhpw68+mRGg= +google.golang.org/api v0.129.0 h1:2XbdjjNfFPXQyufzQVwPf1RRnHH8Den2pfNE2jw7L8w= +google.golang.org/api v0.129.0/go.mod h1:dFjiXlanKwWE3612X97llhsoI36FAoIiRj3aTl5b/zE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 h1:0BOZf6qNozI3pkN3fJLwNubheHJYHhMh91GRFOWWK08= -google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= +google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -264,12 +310,14 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From dbe2eef80c6dd6e377fa1cfc7fb7bc6fa11988bd Mon Sep 17 00:00:00 2001 From: Louis Matthijssen Date: Thu, 22 Jun 2023 17:03:29 +0000 Subject: [PATCH 52/90] Add hostname flag to Docker docs Fixes #4380 --- doc/020_installation.rst | 4 ++++ docker/README.md | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/020_installation.rst b/doc/020_installation.rst index 4d591356d..b53c350b1 100644 --- a/doc/020_installation.rst +++ b/doc/020_installation.rst @@ -265,6 +265,10 @@ binary, you can get it with `docker pull` like this: $ docker pull restic/restic +Restic relies on the hostname for various operations. Make sure to set a static +hostname using `--hostname` when creating a Docker container, otherwise Docker +will assign a random hostname each time. + From Source *********** diff --git a/docker/README.md b/docker/README.md index 1c2c9205c..444aae3dc 100644 --- a/docker/README.md +++ b/docker/README.md @@ -16,9 +16,13 @@ Set environment variable `RESTIC_REPOSITORY` and map volume to directories and files like: ``` -docker run --rm -ti \ +docker run --rm --hostname my-host -ti \ -v $HOME/.restic/passfile:/pass \ -v $HOME/importantdirectory:/data \ -e RESTIC_REPOSITORY=rest:https://user:pass@hostname/ \ restic/restic -p /pass backup /data ``` + +Restic relies on the hostname for various operations. Make sure to set a static +hostname using `--hostname` when creating a Docker container, otherwise Docker +will assign a random hostname each time. From 41a5bf357f7bde96c395b4645012fcb883da159b Mon Sep 17 00:00:00 2001 From: greatroar <61184462+greatroar@users.noreply.github.com> Date: Sun, 2 Jul 2023 20:09:57 +0200 Subject: [PATCH 53/90] cmd, ui: Move size parsing code and make it more robust --- cmd/restic/cmd_check.go | 5 ++-- cmd/restic/cmd_prune.go | 4 +-- cmd/restic/exclude.go | 33 ++----------------------- cmd/restic/exclude_test.go | 48 ------------------------------------ internal/ui/format.go | 41 +++++++++++++++++++++++++++++++ internal/ui/format_test.go | 50 +++++++++++++++++++++++++++++++++++++- 6 files changed, 97 insertions(+), 84 deletions(-) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index b9f3199b2..3c4c9daa9 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -16,6 +16,7 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui" ) var cmdCheck = &cobra.Command{ @@ -97,7 +98,7 @@ func checkFlags(opts CheckOptions) error { } } else { - fileSize, err := parseSizeStr(opts.ReadDataSubset) + fileSize, err := ui.ParseBytes(opts.ReadDataSubset) if err != nil { return argumentError } @@ -363,7 +364,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args if repoSize == 0 { return errors.Fatal("Cannot read from a repository having size 0") } - subsetSize, _ := parseSizeStr(opts.ReadDataSubset) + subsetSize, _ := ui.ParseBytes(opts.ReadDataSubset) if subsetSize > repoSize { subsetSize = repoSize } diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index 1889dffd6..e4c2c7b29 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -81,7 +81,7 @@ func addPruneOptions(c *cobra.Command) { func verifyPruneOptions(opts *PruneOptions) error { opts.MaxRepackBytes = math.MaxUint64 if len(opts.MaxRepackSize) > 0 { - size, err := parseSizeStr(opts.MaxRepackSize) + size, err := ui.ParseBytes(opts.MaxRepackSize) if err != nil { return err } @@ -124,7 +124,7 @@ func verifyPruneOptions(opts *PruneOptions) error { } default: - size, err := parseSizeStr(maxUnused) + size, err := ui.ParseBytes(maxUnused) if err != nil { return errors.Fatalf("invalid number of bytes %q for --max-unused: %v", opts.MaxUnused, err) } diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go index efe6f41e4..095944610 100644 --- a/cmd/restic/exclude.go +++ b/cmd/restic/exclude.go @@ -7,7 +7,6 @@ import ( "io" "os" "path/filepath" - "strconv" "strings" "sync" @@ -17,6 +16,7 @@ import ( "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/textfile" + "github.com/restic/restic/internal/ui" "github.com/spf13/pflag" ) @@ -364,7 +364,7 @@ func rejectResticCache(repo *repository.Repository) (RejectByNameFunc, error) { } func rejectBySize(maxSizeStr string) (RejectFunc, error) { - maxSize, err := parseSizeStr(maxSizeStr) + maxSize, err := ui.ParseBytes(maxSizeStr) if err != nil { return nil, err } @@ -385,35 +385,6 @@ func rejectBySize(maxSizeStr string) (RejectFunc, error) { }, nil } -func parseSizeStr(sizeStr string) (int64, error) { - if sizeStr == "" { - return 0, errors.New("expected size, got empty string") - } - - numStr := sizeStr[:len(sizeStr)-1] - var unit int64 = 1 - - switch sizeStr[len(sizeStr)-1] { - case 'b', 'B': - // use initialized values, do nothing here - case 'k', 'K': - unit = 1024 - case 'm', 'M': - unit = 1024 * 1024 - case 'g', 'G': - unit = 1024 * 1024 * 1024 - case 't', 'T': - unit = 1024 * 1024 * 1024 * 1024 - default: - numStr = sizeStr - } - value, err := strconv.ParseInt(numStr, 10, 64) - if err != nil { - return 0, err - } - return value * unit, nil -} - // readExcludePatternsFromFiles reads all exclude files and returns the list of // exclude patterns. For each line, leading and trailing white space is removed // and comment lines are ignored. For each remaining pattern, environment diff --git a/cmd/restic/exclude_test.go b/cmd/restic/exclude_test.go index 050a083e4..9a24418ae 100644 --- a/cmd/restic/exclude_test.go +++ b/cmd/restic/exclude_test.go @@ -187,54 +187,6 @@ func TestMultipleIsExcludedByFile(t *testing.T) { } } -func TestParseSizeStr(t *testing.T) { - sizeStrTests := []struct { - in string - expected int64 - }{ - {"1024", 1024}, - {"1024b", 1024}, - {"1024B", 1024}, - {"1k", 1024}, - {"100k", 102400}, - {"100K", 102400}, - {"10M", 10485760}, - {"100m", 104857600}, - {"20G", 21474836480}, - {"10g", 10737418240}, - {"2T", 2199023255552}, - {"2t", 2199023255552}, - } - - for _, tt := range sizeStrTests { - actual, err := parseSizeStr(tt.in) - test.OK(t, err) - - if actual != tt.expected { - t.Errorf("parseSizeStr(%s) = %d; expected %d", tt.in, actual, tt.expected) - } - } -} - -func TestParseInvalidSizeStr(t *testing.T) { - invalidSizes := []string{ - "", - " ", - "foobar", - "zzz", - } - - for _, s := range invalidSizes { - v, err := parseSizeStr(s) - if err == nil { - t.Errorf("wanted error for invalid value %q, got nil", s) - } - if v != 0 { - t.Errorf("wanted zero for invalid value %q, got: %v", s, v) - } - } -} - // TestIsExcludedByFileSize is for testing the instance of // --exclude-larger-than parameters func TestIsExcludedByFileSize(t *testing.T) { diff --git a/internal/ui/format.go b/internal/ui/format.go index 34c97703a..d2e0a4d2b 100644 --- a/internal/ui/format.go +++ b/internal/ui/format.go @@ -3,7 +3,10 @@ package ui import ( "bytes" "encoding/json" + "errors" "fmt" + "math/bits" + "strconv" "time" ) @@ -56,6 +59,44 @@ func FormatSeconds(sec uint64) string { return fmt.Sprintf("%d:%02d", min, sec) } +// ParseBytes parses a size in bytes from s. It understands the suffixes +// B, K, M, G and T for powers of 1024. +func ParseBytes(s string) (int64, error) { + if s == "" { + return 0, errors.New("expected size, got empty string") + } + + numStr := s[:len(s)-1] + var unit uint64 = 1 + + switch s[len(s)-1] { + case 'b', 'B': + // use initialized values, do nothing here + case 'k', 'K': + unit = 1024 + case 'm', 'M': + unit = 1024 * 1024 + case 'g', 'G': + unit = 1024 * 1024 * 1024 + case 't', 'T': + unit = 1024 * 1024 * 1024 * 1024 + default: + numStr = s + } + value, err := strconv.ParseInt(numStr, 10, 64) + if err != nil { + return 0, err + } + + hi, lo := bits.Mul64(uint64(value), unit) + value = int64(lo) + if hi != 0 || value < 0 { + return 0, fmt.Errorf("ParseSize: %q: %w", numStr, strconv.ErrRange) + } + + return value, nil +} + func ToJSONString(status interface{}) string { buf := new(bytes.Buffer) err := json.NewEncoder(buf).Encode(status) diff --git a/internal/ui/format_test.go b/internal/ui/format_test.go index b6a1c13d1..4223d4e20 100644 --- a/internal/ui/format_test.go +++ b/internal/ui/format_test.go @@ -1,6 +1,10 @@ package ui -import "testing" +import ( + "testing" + + "github.com/restic/restic/internal/test" +) func TestFormatBytes(t *testing.T) { for _, c := range []struct { @@ -36,3 +40,47 @@ func TestFormatPercent(t *testing.T) { } } } + +func TestParseBytes(t *testing.T) { + for _, tt := range []struct { + in string + expected int64 + }{ + {"1024", 1024}, + {"1024b", 1024}, + {"1024B", 1024}, + {"1k", 1024}, + {"100k", 102400}, + {"100K", 102400}, + {"10M", 10485760}, + {"100m", 104857600}, + {"20G", 21474836480}, + {"10g", 10737418240}, + {"2T", 2199023255552}, + {"2t", 2199023255552}, + {"9223372036854775807", 1<<63 - 1}, + } { + actual, err := ParseBytes(tt.in) + test.OK(t, err) + test.Equals(t, tt.expected, actual) + } +} + +func TestParseBytesInvalid(t *testing.T) { + for _, s := range []string{ + "", + " ", + "foobar", + "zzz", + "18446744073709551615", // 1<<64-1. + "9223372036854775807k", // 1<<63-1 kiB. + "9999999999999M", + "99999999999999999999", + } { + v, err := ParseBytes(s) + if err == nil { + t.Errorf("wanted error for invalid value %q, got nil", s) + } + test.Equals(t, int64(0), v) + } +} From 068b115abc8ed549fc966a989b9c6ab7d0ce2085 Mon Sep 17 00:00:00 2001 From: arjunajesh <34989598+arjunajesh@users.noreply.github.com> Date: Sat, 24 Jun 2023 20:06:54 -0400 Subject: [PATCH 54/90] added azure domain parameter --- changelog/unreleased/issue-2468 | 6 ++++++ doc/030_preparing_a_new_repo.rst | 7 +++++++ doc/040_backup.rst | 1 + internal/backend/azure/azure.go | 8 +++++++- internal/backend/azure/config.go | 14 +++++++++----- 5 files changed, 30 insertions(+), 6 deletions(-) create mode 100644 changelog/unreleased/issue-2468 diff --git a/changelog/unreleased/issue-2468 b/changelog/unreleased/issue-2468 new file mode 100644 index 000000000..5a2cafc85 --- /dev/null +++ b/changelog/unreleased/issue-2468 @@ -0,0 +1,6 @@ +Enhancement: Add support for non-global Azure clouds + +Restic backups on azure only worked for storages on the global domain `core.windows.net`. This meant that backups to other domains such as Azure China (`core.chinacloudapi.cn') were not supported. Restic now allows overriding the global domain using the environment variable `AZURE_ENDPOINT_SUFFIX'. + +https://github.com/restic/restic/issues/2468 +https://github.com/restic/restic/pull/4387 diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index a871ee507..52b20a788 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -537,6 +537,13 @@ or $ export AZURE_ACCOUNT_NAME= $ export AZURE_ACCOUNT_SAS= +Restic will use Azure's global domain ``core.windows.net`` by default. You can specify other +domains to be used like so: + +.. code-block:: console + + $export AZURE_ENDPOINT_SUFFIX= + Afterwards you can initialize a repository in a container called ``foo`` in the root path like this: diff --git a/doc/040_backup.rst b/doc/040_backup.rst index c09307da0..c52fec8c4 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -614,6 +614,7 @@ environment variables. The following lists these environment variables: AZURE_ACCOUNT_NAME Account name for Azure AZURE_ACCOUNT_KEY Account key for Azure AZURE_ACCOUNT_SAS Shared access signatures (SAS) for Azure + AZURE_ENDPOINT_SUFFIX Domain of Azure Storage (default: core.windows.net) GOOGLE_PROJECT_ID Project ID for Google Cloud Storage GOOGLE_APPLICATION_CREDENTIALS Application Credentials for Google Cloud Storage (e.g. $HOME/.config/gs-secret-restic-key.json) diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index a9267a945..661dd505d 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -53,7 +53,13 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { var client *azContainer.Client var err error - url := fmt.Sprintf("https://%s.blob.core.windows.net/%s", cfg.AccountName, cfg.Container) + var endpointSuffix string + if cfg.EndpointSuffix != "" { + endpointSuffix = cfg.EndpointSuffix + } else { + endpointSuffix = "core.windows.net" + } + url := fmt.Sprintf("https://%s.blob.%s/%s", cfg.AccountName, endpointSuffix, cfg.Container) opts := &azContainer.ClientOptions{ ClientOptions: azcore.ClientOptions{ Transport: &http.Client{Transport: rt}, diff --git a/internal/backend/azure/config.go b/internal/backend/azure/config.go index 6786ec626..5284572e9 100644 --- a/internal/backend/azure/config.go +++ b/internal/backend/azure/config.go @@ -13,11 +13,12 @@ import ( // Config contains all configuration necessary to connect to an azure compatible // server. type Config struct { - AccountName string - AccountSAS options.SecretString - AccountKey options.SecretString - Container string - Prefix string + AccountName string + AccountSAS options.SecretString + AccountKey options.SecretString + EndpointSuffix string + Container string + Prefix string Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"` } @@ -71,4 +72,7 @@ func (cfg *Config) ApplyEnvironment(prefix string) { if cfg.AccountSAS.String() == "" { cfg.AccountSAS = options.NewSecretString(os.Getenv(prefix + "AZURE_ACCOUNT_SAS")) } + if cfg.EndpointSuffix == "" { + cfg.EndpointSuffix = os.Getenv("AZURE_ENDPOINT_SUFFIX") + } } From e36d17a6f852e4b19359ae82db4e7321698f4eb7 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 7 Jul 2023 23:09:44 +0200 Subject: [PATCH 55/90] azure: tweak documentation for endpoint suffix --- changelog/unreleased/issue-2468 | 6 +++++- doc/030_preparing_a_new_repo.rst | 6 +++--- doc/040_backup.rst | 2 +- internal/backend/azure/config.go | 3 ++- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/changelog/unreleased/issue-2468 b/changelog/unreleased/issue-2468 index 5a2cafc85..56555a136 100644 --- a/changelog/unreleased/issue-2468 +++ b/changelog/unreleased/issue-2468 @@ -1,6 +1,10 @@ Enhancement: Add support for non-global Azure clouds -Restic backups on azure only worked for storages on the global domain `core.windows.net`. This meant that backups to other domains such as Azure China (`core.chinacloudapi.cn') were not supported. Restic now allows overriding the global domain using the environment variable `AZURE_ENDPOINT_SUFFIX'. +Restic backups on Azure only supported storages using the global domain +`core.windows.net`. This meant that backups to other domains such as Azure +China (`core.chinacloudapi.cn') or Azure Germany (`core.cloudapi.de`) were +not supported. Restic now allows overriding the global domain using the +environment variable `AZURE_ENDPOINT_SUFFIX'. https://github.com/restic/restic/issues/2468 https://github.com/restic/restic/pull/4387 diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index 52b20a788..c944264c8 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -537,12 +537,12 @@ or $ export AZURE_ACCOUNT_NAME= $ export AZURE_ACCOUNT_SAS= -Restic will use Azure's global domain ``core.windows.net`` by default. You can specify other -domains to be used like so: +Restic will by default use Azure's global domain ``core.windows.net`` as endpoint suffix. +You can specify other suffixes as follows: .. code-block:: console - $export AZURE_ENDPOINT_SUFFIX= + $ export AZURE_ENDPOINT_SUFFIX= Afterwards you can initialize a repository in a container called ``foo`` in the root path like this: diff --git a/doc/040_backup.rst b/doc/040_backup.rst index c52fec8c4..7856875f0 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -614,7 +614,7 @@ environment variables. The following lists these environment variables: AZURE_ACCOUNT_NAME Account name for Azure AZURE_ACCOUNT_KEY Account key for Azure AZURE_ACCOUNT_SAS Shared access signatures (SAS) for Azure - AZURE_ENDPOINT_SUFFIX Domain of Azure Storage (default: core.windows.net) + AZURE_ENDPOINT_SUFFIX Endpoint suffix for Azure Storage (default: core.windows.net) GOOGLE_PROJECT_ID Project ID for Google Cloud Storage GOOGLE_APPLICATION_CREDENTIALS Application Credentials for Google Cloud Storage (e.g. $HOME/.config/gs-secret-restic-key.json) diff --git a/internal/backend/azure/config.go b/internal/backend/azure/config.go index 5284572e9..d819b35aa 100644 --- a/internal/backend/azure/config.go +++ b/internal/backend/azure/config.go @@ -72,7 +72,8 @@ func (cfg *Config) ApplyEnvironment(prefix string) { if cfg.AccountSAS.String() == "" { cfg.AccountSAS = options.NewSecretString(os.Getenv(prefix + "AZURE_ACCOUNT_SAS")) } + if cfg.EndpointSuffix == "" { - cfg.EndpointSuffix = os.Getenv("AZURE_ENDPOINT_SUFFIX") + cfg.EndpointSuffix = os.Getenv(prefix + "AZURE_ENDPOINT_SUFFIX") } } From bbac74b17225642380a52da1d96ace6c401285d7 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 19 Jun 2023 19:30:41 +0200 Subject: [PATCH 56/90] add program version to snapshot --- changelog/unreleased/issue-4188 | 7 +++++++ cmd/restic/cmd_backup.go | 1 + internal/archiver/archiver.go | 2 ++ internal/restic/snapshot.go | 2 ++ 4 files changed, 12 insertions(+) create mode 100644 changelog/unreleased/issue-4188 diff --git a/changelog/unreleased/issue-4188 b/changelog/unreleased/issue-4188 new file mode 100644 index 000000000..377468a76 --- /dev/null +++ b/changelog/unreleased/issue-4188 @@ -0,0 +1,7 @@ +Enhancement: Include client version in snapshot metadata + +The client version number is now included in the `program_version` field of a snapshot. +It can be inspected by using either `restic cat snapshot snapshotID` or `restic snapshots --json`. + +https://github.com/restic/restic/issues/4188 +https://github.com/restic/restic/pull/4378 diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index f12f70e39..d7e899eaf 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -645,6 +645,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter Time: timeStamp, Hostname: opts.Host, ParentSnapshot: parentSnapshot, + ProgramVersion: "restic " + version, } if !gopts.JSON { diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index 3c1cc33d0..98819d797 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -680,6 +680,7 @@ type SnapshotOptions struct { Excludes []string Time time.Time ParentSnapshot *restic.Snapshot + ProgramVersion string } // loadParentTree loads a tree referenced by snapshot id. If id is null, nil is returned. @@ -796,6 +797,7 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps return nil, restic.ID{}, err } + sn.ProgramVersion = opts.ProgramVersion sn.Excludes = opts.Excludes if opts.ParentSnapshot != nil { sn.Parent = opts.ParentSnapshot.ID() diff --git a/internal/restic/snapshot.go b/internal/restic/snapshot.go index 1f6e4534b..13e795ec8 100644 --- a/internal/restic/snapshot.go +++ b/internal/restic/snapshot.go @@ -25,6 +25,8 @@ type Snapshot struct { Tags []string `json:"tags,omitempty"` Original *ID `json:"original,omitempty"` + ProgramVersion string `json:"program_version,omitempty"` + id *ID // plaintext ID, used during restore } From 389f6ee74c717e28b99b9418021bb43f0cc972eb Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 7 Jul 2023 23:33:23 +0200 Subject: [PATCH 57/90] backup: add minimal test for program versioni --- cmd/restic/cmd_backup_integration_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cmd/restic/cmd_backup_integration_test.go b/cmd/restic/cmd_backup_integration_test.go index b81db21e6..fb7bef633 100644 --- a/cmd/restic/cmd_backup_integration_test.go +++ b/cmd/restic/cmd_backup_integration_test.go @@ -440,6 +440,22 @@ func TestBackupTags(t *testing.T) { "expected parent to be %v, got %v", parent.ID, newest.Parent) } +func TestBackupProgramVersion(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts) + newest, _ := testRunSnapshots(t, env.gopts) + + if newest == nil { + t.Fatal("expected a backup, got nil") + } + resticVersion := "restic " + version + rtest.Assert(t, newest.ProgramVersion == resticVersion, + "expected %v, got %v", resticVersion, newest.ProgramVersion) +} + func TestQuietBackup(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() From e703e89e9bb72abd9d1e9f90466c891229d3c0fd Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 7 Jul 2023 23:36:57 +0200 Subject: [PATCH 58/90] add changelog for program version --- changelog/unreleased/issue-4188 | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/changelog/unreleased/issue-4188 b/changelog/unreleased/issue-4188 index 377468a76..dbb26f733 100644 --- a/changelog/unreleased/issue-4188 +++ b/changelog/unreleased/issue-4188 @@ -1,7 +1,8 @@ -Enhancement: Include client version in snapshot metadata +Enhancement: `backup` includes restic version in snapshot metadata -The client version number is now included in the `program_version` field of a snapshot. -It can be inspected by using either `restic cat snapshot snapshotID` or `restic snapshots --json`. +The restic version used backup the snapshot is now included in its metadata. +The program version is shown when inspecting a snapshot using `restic cat +snapshot ` or `restic snapshots --json`. https://github.com/restic/restic/issues/4188 https://github.com/restic/restic/pull/4378 From 3a32c4e59f9c63dfe5dc1b4fd2b89329159c8516 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 17 Jun 2023 17:48:54 +0200 Subject: [PATCH 59/90] document how to analyze performance / memory usage issues --- CONTRIBUTING.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 36a7c0695..e0f2e2c08 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -88,10 +88,40 @@ Then use the `go` tool to build restic: $ ./restic version restic 0.14.0-dev (compiled manually) compiled with go1.19 on linux/amd64 +To create a debug build use: + + $ go build -tags debug ./cmd/restic + You can run all tests with the following command: $ go test ./... + +Performance and Memory Usage Issues +=================================== + +Debug builds of restic support the `--block-profile`, `--cpu-profile`, +`--mem-profile`, and `--trace-profile` options which collect performance data +that later on can be analyzed using the go tools: + + $ restic --cpu-profile . [...] + $ go tool pprof -http localhost:12345 cpu.pprof + +To analyze a trace profile use `go tool trace -http=localhost:12345 trace.out`. + +As the memory usage of restic changes over time, it may be useful to capture a +snapshot of the current heap. This is possible using then `--listen-profile` +option. Then while restic runs you can query and afterwards analyze the heap statistics. + + $ restic --listen-profile localhost:12345 [...] + $ curl http://localhost:12345/debug/pprof/heap -o heap.pprof + $ go tool pprof -http localhost:12345 heap.pprof + +Further useful tools are setting the environment variable `GODEBUG=gctrace=1`, +which provides information about garbage collector runs. For a graphical variant +combine this with gcvis. + + Providing Patches ================= From c1578a2035406805b63c52647870274505d80e88 Mon Sep 17 00:00:00 2001 From: arjunajesh <34989598+arjunajesh@users.noreply.github.com> Date: Thu, 22 Jun 2023 16:10:41 -0400 Subject: [PATCH 60/90] certificates can be passed through env vars --- changelog/unreleased/issue-1926 | 6 ++++++ cmd/restic/global.go | 4 +++- doc/040_backup.rst | 2 ++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/issue-1926 diff --git a/changelog/unreleased/issue-1926 b/changelog/unreleased/issue-1926 new file mode 100644 index 000000000..8d16bb8db --- /dev/null +++ b/changelog/unreleased/issue-1926 @@ -0,0 +1,6 @@ +Enhancemnet: Certificates can be passed through environment variables + +Restic will now read the paths to the certificates from the environment variables `RESTIC_CACERT` or `RESTIC_TLS_CLIENT_CERT` if `--cacert` or `--tls-client-cert` is not specified. + +https://github.com/restic/restic/issues/1926 +https://github.com/restic/restic/pull/4384 diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 823a82e36..3f55e1cbe 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -135,7 +135,7 @@ func init() { f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache `directory`. (default: use system default cache directory)") f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache") f.StringSliceVar(&globalOptions.RootCertFilenames, "cacert", nil, "`file` to load root certificates from (default: use system certificates)") - f.StringVar(&globalOptions.TLSClientCertKeyFilename, "tls-client-cert", "", "path to a `file` containing PEM encoded TLS client certificate and private key") + f.StringVar(&globalOptions.TLSClientCertKeyFilename, "tls-client-cert", "", "path to a `file` containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT)") f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)") f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)") @@ -151,6 +151,8 @@ func init() { globalOptions.PasswordFile = os.Getenv("RESTIC_PASSWORD_FILE") globalOptions.KeyHint = os.Getenv("RESTIC_KEY_HINT") globalOptions.PasswordCommand = os.Getenv("RESTIC_PASSWORD_COMMAND") + globalOptions.RootCertFilenames = strings.Split(os.Getenv("RESTIC_CACERT"), ",") + globalOptions.TLSClientCertKeyFilename = os.Getenv("RESTIC_TLS_CLIENT_CERT") comp := os.Getenv("RESTIC_COMPRESSION") if comp != "" { // ignore error as there's no good way to handle it diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 7856875f0..8ab2a50d6 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -567,6 +567,8 @@ environment variables. The following lists these environment variables: RESTIC_PASSWORD The actual password for the repository RESTIC_PASSWORD_COMMAND Command printing the password for the repository to stdout RESTIC_KEY_HINT ID of key to try decrypting first, before other keys + RESTIC_CACERT Location(s) of certificate file(s), comma seperated if multiple (replaces --cacert) + RESTIC_TLS_CLIENT_CERT Location of TLS client certificate and private key (replaces --tls-client-cert) RESTIC_CACHE_DIR Location of the cache directory RESTIC_COMPRESSION Compression mode (only available for repository format version 2) RESTIC_PROGRESS_FPS Frames per second by which the progress bar is updated From cc3c218bafc02b7ba230cc387845f21d9fbe4069 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 8 Jul 2023 09:44:20 +0200 Subject: [PATCH 61/90] small cleanups for certificate environment variables --- changelog/unreleased/issue-1926 | 6 ++++-- cmd/restic/global.go | 2 +- doc/040_backup.rst | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/changelog/unreleased/issue-1926 b/changelog/unreleased/issue-1926 index 8d16bb8db..9f172b1f8 100644 --- a/changelog/unreleased/issue-1926 +++ b/changelog/unreleased/issue-1926 @@ -1,6 +1,8 @@ -Enhancemnet: Certificates can be passed through environment variables +Enhancement: Certificates can be passed through environment variables -Restic will now read the paths to the certificates from the environment variables `RESTIC_CACERT` or `RESTIC_TLS_CLIENT_CERT` if `--cacert` or `--tls-client-cert` is not specified. +Restic will now read the paths to the certificates from the environment +variables `RESTIC_CACERT` or `RESTIC_TLS_CLIENT_CERT` if `--cacert` or +`--tls-client-cert` are not specified. https://github.com/restic/restic/issues/1926 https://github.com/restic/restic/pull/4384 diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 3f55e1cbe..487fa9673 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -134,7 +134,7 @@ func init() { f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it") f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache `directory`. (default: use system default cache directory)") f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache") - f.StringSliceVar(&globalOptions.RootCertFilenames, "cacert", nil, "`file` to load root certificates from (default: use system certificates)") + f.StringSliceVar(&globalOptions.RootCertFilenames, "cacert", nil, "`file` to load root certificates from (default: use system certificates or $RESTIC_CACERT)") f.StringVar(&globalOptions.TLSClientCertKeyFilename, "tls-client-cert", "", "path to a `file` containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT)") f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)") f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 8ab2a50d6..b01683929 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -567,7 +567,7 @@ environment variables. The following lists these environment variables: RESTIC_PASSWORD The actual password for the repository RESTIC_PASSWORD_COMMAND Command printing the password for the repository to stdout RESTIC_KEY_HINT ID of key to try decrypting first, before other keys - RESTIC_CACERT Location(s) of certificate file(s), comma seperated if multiple (replaces --cacert) + RESTIC_CACERT Location(s) of certificate file(s), comma separated if multiple (replaces --cacert) RESTIC_TLS_CLIENT_CERT Location of TLS client certificate and private key (replaces --tls-client-cert) RESTIC_CACHE_DIR Location of the cache directory RESTIC_COMPRESSION Compression mode (only available for repository format version 2) From f3c3b0f377ecfed00af10957e145d77ff275ce7d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 8 Jul 2023 17:41:45 +0200 Subject: [PATCH 62/90] fuse: deduplicate xattr code --- internal/fuse/dir.go | 13 ++----------- internal/fuse/file.go | 13 ++----------- internal/fuse/link.go | 15 ++------------- internal/fuse/xattr.go | 24 ++++++++++++++++++++++++ 4 files changed, 30 insertions(+), 35 deletions(-) create mode 100644 internal/fuse/xattr.go diff --git a/internal/fuse/dir.go b/internal/fuse/dir.go index 7dc157b7e..242b4b03e 100644 --- a/internal/fuse/dir.go +++ b/internal/fuse/dir.go @@ -222,19 +222,10 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) { } func (d *dir) Listxattr(_ context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error { - debug.Log("Listxattr(%v, %v)", d.node.Name, req.Size) - for _, attr := range d.node.ExtendedAttributes { - resp.Append(attr.Name) - } + nodeToXattrList(d.node, req, resp) return nil } func (d *dir) Getxattr(_ context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { - debug.Log("Getxattr(%v, %v, %v)", d.node.Name, req.Name, req.Size) - attrval := d.node.GetExtendedAttribute(req.Name) - if attrval != nil { - resp.Xattr = attrval - return nil - } - return fuse.ErrNoXattr + return nodeGetXattr(d.node, req, resp) } diff --git a/internal/fuse/file.go b/internal/fuse/file.go index fd9d8ccc2..aec39273a 100644 --- a/internal/fuse/file.go +++ b/internal/fuse/file.go @@ -167,19 +167,10 @@ func (f *openFile) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.R } func (f *file) Listxattr(_ context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error { - debug.Log("Listxattr(%v, %v)", f.node.Name, req.Size) - for _, attr := range f.node.ExtendedAttributes { - resp.Append(attr.Name) - } + nodeToXattrList(f.node, req, resp) return nil } func (f *file) Getxattr(_ context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { - debug.Log("Getxattr(%v, %v, %v)", f.node.Name, req.Name, req.Size) - attrval := f.node.GetExtendedAttribute(req.Name) - if attrval != nil { - resp.Xattr = attrval - return nil - } - return fuse.ErrNoXattr + return nodeGetXattr(f.node, req, resp) } diff --git a/internal/fuse/link.go b/internal/fuse/link.go index 43b00a855..3aea8b06e 100644 --- a/internal/fuse/link.go +++ b/internal/fuse/link.go @@ -6,8 +6,6 @@ package fuse import ( "context" - "github.com/restic/restic/internal/debug" - "github.com/anacrolix/fuse" "github.com/anacrolix/fuse/fs" "github.com/restic/restic/internal/restic" @@ -50,19 +48,10 @@ func (l *link) Attr(_ context.Context, a *fuse.Attr) error { } func (l *link) Listxattr(_ context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error { - debug.Log("Listxattr(%v, %v)", l.node.Name, req.Size) - for _, attr := range l.node.ExtendedAttributes { - resp.Append(attr.Name) - } + nodeToXattrList(l.node, req, resp) return nil } func (l *link) Getxattr(_ context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { - debug.Log("Getxattr(%v, %v, %v)", l.node.Name, req.Name, req.Size) - attrval := l.node.GetExtendedAttribute(req.Name) - if attrval != nil { - resp.Xattr = attrval - return nil - } - return fuse.ErrNoXattr + return nodeGetXattr(l.node, req, resp) } diff --git a/internal/fuse/xattr.go b/internal/fuse/xattr.go new file mode 100644 index 000000000..f208938a6 --- /dev/null +++ b/internal/fuse/xattr.go @@ -0,0 +1,24 @@ +package fuse + +import ( + "github.com/anacrolix/fuse" + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/restic" +) + +func nodeToXattrList(node *restic.Node, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) { + debug.Log("Listxattr(%v, %v)", node.Name, req.Size) + for _, attr := range node.ExtendedAttributes { + resp.Append(attr.Name) + } +} + +func nodeGetXattr(node *restic.Node, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { + debug.Log("Getxattr(%v, %v, %v)", node.Name, req.Name, req.Size) + attrval := node.GetExtendedAttribute(req.Name) + if attrval != nil { + resp.Xattr = attrval + return nil + } + return fuse.ErrNoXattr +} From 1f1e50f49efd5344ac0d01e206ec4232651a00bc Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 8 Jul 2023 18:02:17 +0200 Subject: [PATCH 63/90] fuse: add test for symlink xattr --- internal/fuse/fuse_test.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/internal/fuse/fuse_test.go b/internal/fuse/fuse_test.go index 9ca1ec0c6..ccdd2f774 100644 --- a/internal/fuse/fuse_test.go +++ b/internal/fuse/fuse_test.go @@ -271,6 +271,31 @@ func TestInodeFromNode(t *testing.T) { rtest.Assert(t, inoA != inoAbb, "inode(a/b/b) = inode(a)") } +func TestLink(t *testing.T) { + node := &restic.Node{Name: "foo.txt", Type: "symlink", Links: 1, LinkTarget: "dst", ExtendedAttributes: []restic.ExtendedAttribute{ + {Name: "foo", Value: []byte("bar")}, + }} + + lnk, err := newLink(&Root{}, 42, node) + rtest.OK(t, err) + target, err := lnk.Readlink(context.TODO(), nil) + rtest.OK(t, err) + rtest.Equals(t, node.LinkTarget, target) + + exp := &fuse.ListxattrResponse{} + exp.Append("foo") + resp := &fuse.ListxattrResponse{} + rtest.OK(t, lnk.Listxattr(context.TODO(), &fuse.ListxattrRequest{}, resp)) + rtest.Equals(t, exp.Xattr, resp.Xattr) + + getResp := &fuse.GetxattrResponse{} + rtest.OK(t, lnk.Getxattr(context.TODO(), &fuse.GetxattrRequest{Name: "foo"}, getResp)) + rtest.Equals(t, node.ExtendedAttributes[0].Value, getResp.Xattr) + + err = lnk.Getxattr(context.TODO(), &fuse.GetxattrRequest{Name: "invalid"}, nil) + rtest.Assert(t, err != nil, "missing error on reading invalid xattr") +} + var sink uint64 func BenchmarkInode(b *testing.B) { From 4a5ae2ba8451d4a8c64ace57dc4da8e1b36fbd69 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 8 Jul 2023 18:18:13 +0200 Subject: [PATCH 64/90] restic: test NodeFromFileInfo for symlinks --- internal/restic/node_unix_test.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/internal/restic/node_unix_test.go b/internal/restic/node_unix_test.go index c4fef3710..374326bf7 100644 --- a/internal/restic/node_unix_test.go +++ b/internal/restic/node_unix_test.go @@ -5,10 +5,13 @@ package restic import ( "os" + "path/filepath" "runtime" "syscall" "testing" "time" + + rtest "github.com/restic/restic/internal/test" ) func stat(t testing.TB, filename string) (fi os.FileInfo, ok bool) { @@ -25,6 +28,7 @@ func stat(t testing.TB, filename string) (fi os.FileInfo, ok bool) { } func checkFile(t testing.TB, stat *syscall.Stat_t, node *Node) { + t.Helper() if uint32(node.Mode.Perm()) != uint32(stat.Mode&0777) { t.Errorf("Mode does not match, want %v, got %v", stat.Mode&0777, node.Mode) } @@ -37,7 +41,7 @@ func checkFile(t testing.TB, stat *syscall.Stat_t, node *Node) { t.Errorf("Dev does not match, want %v, got %v", stat.Dev, node.DeviceID) } - if node.Size != uint64(stat.Size) { + if node.Size != uint64(stat.Size) && node.Type != "symlink" { t.Errorf("Size does not match, want %v, got %v", stat.Size, node.Size) } @@ -83,6 +87,10 @@ func checkDevice(t testing.TB, stat *syscall.Stat_t, node *Node) { } func TestNodeFromFileInfo(t *testing.T) { + tmp := t.TempDir() + symlink := filepath.Join(tmp, "symlink") + rtest.OK(t, os.Symlink("target", symlink)) + type Test struct { filename string canSkip bool @@ -90,6 +98,7 @@ func TestNodeFromFileInfo(t *testing.T) { var tests = []Test{ {"node_test.go", false}, {"/dev/sda", true}, + {symlink, false}, } // on darwin, users are not permitted to list the extended attributes of @@ -125,7 +134,7 @@ func TestNodeFromFileInfo(t *testing.T) { } switch node.Type { - case "file": + case "file", "symlink": checkFile(t, s, node) case "dev", "chardev": checkFile(t, s, node) From cc84884d2e21f3847830fa7a507946967feb9101 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 8 Jul 2023 18:49:21 +0200 Subject: [PATCH 65/90] restic: basic xattr test for files/dirs --- internal/restic/node_test.go | 62 +++++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 11 deletions(-) diff --git a/internal/restic/node_test.go b/internal/restic/node_test.go index 60342e9a4..c89bc6ac3 100644 --- a/internal/restic/node_test.go +++ b/internal/restic/node_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "reflect" "runtime" "testing" "time" @@ -163,22 +164,56 @@ var nodeTests = []restic.Node{ AccessTime: parseTime("2005-05-14 21:07:04.222"), ChangeTime: parseTime("2005-05-14 21:07:05.333"), }, + { + Name: "testXattrFile", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0604, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + ExtendedAttributes: []restic.ExtendedAttribute{ + {"user.foo", []byte("bar")}, + }, + }, + { + Name: "testXattrDir", + Type: "dir", + Subtree: nil, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0750 | os.ModeDir, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + ExtendedAttributes: []restic.ExtendedAttribute{ + {"user.foo", []byte("bar")}, + }, + }, } func TestNodeRestoreAt(t *testing.T) { - tempdir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-") - rtest.OK(t, err) - - defer func() { - if rtest.TestCleanupTempDirs { - rtest.RemoveAll(t, tempdir) - } else { - t.Logf("leaving tempdir at %v", tempdir) - } - }() + tempdir := t.TempDir() for _, test := range nodeTests { - nodePath := filepath.Join(tempdir, test.Name) + var nodePath string + if test.ExtendedAttributes != nil { + if runtime.GOOS == "windows" { + // restic does not support xattrs on windows + return + } + + // tempdir might be backed by a filesystem that does not support + // extended attributes + nodePath = test.Name + defer func() { + _ = os.Remove(nodePath) + }() + } else { + nodePath = filepath.Join(tempdir, test.Name) + } rtest.OK(t, test.CreateAt(context.TODO(), nodePath, nil)) rtest.OK(t, test.RestoreMetadata(nodePath)) @@ -215,6 +250,11 @@ func TestNodeRestoreAt(t *testing.T) { AssertFsTimeEqual(t, "AccessTime", test.Type, test.AccessTime, n2.AccessTime) AssertFsTimeEqual(t, "ModTime", test.Type, test.ModTime, n2.ModTime) + if len(n2.ExtendedAttributes) == 0 { + n2.ExtendedAttributes = nil + } + rtest.Assert(t, reflect.DeepEqual(test.ExtendedAttributes, n2.ExtendedAttributes), + "%v: xattrs don't match (%v != %v)", test.Name, test.ExtendedAttributes, n2.ExtendedAttributes) } } From ea9ad77e05918731625500a4335f285f5156dc79 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 8 Jul 2023 18:49:47 +0200 Subject: [PATCH 66/90] restic: refactor node test --- internal/restic/node_test.go | 100 ++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 49 deletions(-) diff --git a/internal/restic/node_test.go b/internal/restic/node_test.go index c89bc6ac3..45ccd790c 100644 --- a/internal/restic/node_test.go +++ b/internal/restic/node_test.go @@ -198,63 +198,65 @@ func TestNodeRestoreAt(t *testing.T) { tempdir := t.TempDir() for _, test := range nodeTests { - var nodePath string - if test.ExtendedAttributes != nil { - if runtime.GOOS == "windows" { - // restic does not support xattrs on windows - return + t.Run("", func(t *testing.T) { + var nodePath string + if test.ExtendedAttributes != nil { + if runtime.GOOS == "windows" { + // restic does not support xattrs on windows + return + } + + // tempdir might be backed by a filesystem that does not support + // extended attributes + nodePath = test.Name + defer func() { + _ = os.Remove(nodePath) + }() + } else { + nodePath = filepath.Join(tempdir, test.Name) + } + rtest.OK(t, test.CreateAt(context.TODO(), nodePath, nil)) + rtest.OK(t, test.RestoreMetadata(nodePath)) + + if test.Type == "dir" { + rtest.OK(t, test.RestoreTimestamps(nodePath)) } - // tempdir might be backed by a filesystem that does not support - // extended attributes - nodePath = test.Name - defer func() { - _ = os.Remove(nodePath) - }() - } else { - nodePath = filepath.Join(tempdir, test.Name) - } - rtest.OK(t, test.CreateAt(context.TODO(), nodePath, nil)) - rtest.OK(t, test.RestoreMetadata(nodePath)) + fi, err := os.Lstat(nodePath) + rtest.OK(t, err) - if test.Type == "dir" { - rtest.OK(t, test.RestoreTimestamps(nodePath)) - } + n2, err := restic.NodeFromFileInfo(nodePath, fi) + rtest.OK(t, err) - fi, err := os.Lstat(nodePath) - rtest.OK(t, err) + rtest.Assert(t, test.Name == n2.Name, + "%v: name doesn't match (%v != %v)", test.Type, test.Name, n2.Name) + rtest.Assert(t, test.Type == n2.Type, + "%v: type doesn't match (%v != %v)", test.Type, test.Type, n2.Type) + rtest.Assert(t, test.Size == n2.Size, + "%v: size doesn't match (%v != %v)", test.Size, test.Size, n2.Size) - n2, err := restic.NodeFromFileInfo(nodePath, fi) - rtest.OK(t, err) - - rtest.Assert(t, test.Name == n2.Name, - "%v: name doesn't match (%v != %v)", test.Type, test.Name, n2.Name) - rtest.Assert(t, test.Type == n2.Type, - "%v: type doesn't match (%v != %v)", test.Type, test.Type, n2.Type) - rtest.Assert(t, test.Size == n2.Size, - "%v: size doesn't match (%v != %v)", test.Size, test.Size, n2.Size) - - if runtime.GOOS != "windows" { - rtest.Assert(t, test.UID == n2.UID, - "%v: UID doesn't match (%v != %v)", test.Type, test.UID, n2.UID) - rtest.Assert(t, test.GID == n2.GID, - "%v: GID doesn't match (%v != %v)", test.Type, test.GID, n2.GID) - if test.Type != "symlink" { - // On OpenBSD only root can set sticky bit (see sticky(8)). - if runtime.GOOS != "openbsd" && runtime.GOOS != "netbsd" && runtime.GOOS != "solaris" && test.Name == "testSticky" { - rtest.Assert(t, test.Mode == n2.Mode, - "%v: mode doesn't match (0%o != 0%o)", test.Type, test.Mode, n2.Mode) + if runtime.GOOS != "windows" { + rtest.Assert(t, test.UID == n2.UID, + "%v: UID doesn't match (%v != %v)", test.Type, test.UID, n2.UID) + rtest.Assert(t, test.GID == n2.GID, + "%v: GID doesn't match (%v != %v)", test.Type, test.GID, n2.GID) + if test.Type != "symlink" { + // On OpenBSD only root can set sticky bit (see sticky(8)). + if runtime.GOOS != "openbsd" && runtime.GOOS != "netbsd" && runtime.GOOS != "solaris" && test.Name == "testSticky" { + rtest.Assert(t, test.Mode == n2.Mode, + "%v: mode doesn't match (0%o != 0%o)", test.Type, test.Mode, n2.Mode) + } } } - } - AssertFsTimeEqual(t, "AccessTime", test.Type, test.AccessTime, n2.AccessTime) - AssertFsTimeEqual(t, "ModTime", test.Type, test.ModTime, n2.ModTime) - if len(n2.ExtendedAttributes) == 0 { - n2.ExtendedAttributes = nil - } - rtest.Assert(t, reflect.DeepEqual(test.ExtendedAttributes, n2.ExtendedAttributes), - "%v: xattrs don't match (%v != %v)", test.Name, test.ExtendedAttributes, n2.ExtendedAttributes) + AssertFsTimeEqual(t, "AccessTime", test.Type, test.AccessTime, n2.AccessTime) + AssertFsTimeEqual(t, "ModTime", test.Type, test.ModTime, n2.ModTime) + if len(n2.ExtendedAttributes) == 0 { + n2.ExtendedAttributes = nil + } + rtest.Assert(t, reflect.DeepEqual(test.ExtendedAttributes, n2.ExtendedAttributes), + "%v: xattrs don't match (%v != %v)", test.Name, test.ExtendedAttributes, n2.ExtendedAttributes) + }) } } From 9d44682e3ebeb178dfb9d30fb9e0f282b6b56e02 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 8 Jul 2023 22:40:22 +0200 Subject: [PATCH 67/90] fuse: fix windows build --- internal/fuse/xattr.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/fuse/xattr.go b/internal/fuse/xattr.go index f208938a6..e534c3c0e 100644 --- a/internal/fuse/xattr.go +++ b/internal/fuse/xattr.go @@ -1,3 +1,6 @@ +//go:build darwin || freebsd || linux +// +build darwin freebsd linux + package fuse import ( From 325fa916b56df742307f448abdc8a4e780fe6466 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 7 Jun 2023 21:59:28 +0200 Subject: [PATCH 68/90] stats: Add debug mode to collect repository statistics --- cmd/restic/cmd_stats.go | 154 +++++++++++++++++++++++++++++++++++ cmd/restic/cmd_stats_test.go | 62 ++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 cmd/restic/cmd_stats_test.go diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 7032bdef8..a7ecd438f 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -5,11 +5,15 @@ import ( "encoding/json" "fmt" "path/filepath" + "strings" + "github.com/restic/chunker" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/crypto" + "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui" + "github.com/restic/restic/internal/ui/table" "github.com/restic/restic/internal/walker" "github.com/minio/sha256-simd" @@ -99,6 +103,10 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args return err } + if opts.countMode == countModeDebug { + return statsDebug(ctx, repo) + } + if !gopts.JSON { Printf("scanning...\n") } @@ -291,6 +299,7 @@ func verifyStatsInput(opts StatsOptions) error { case countModeUniqueFilesByContents: case countModeBlobsPerFile: case countModeRawData: + case countModeDebug: default: return fmt.Errorf("unknown counting mode: %s (use the -h flag to get a list of supported modes)", opts.countMode) } @@ -335,4 +344,149 @@ const ( countModeUniqueFilesByContents = "files-by-contents" countModeBlobsPerFile = "blobs-per-file" countModeRawData = "raw-data" + countModeDebug = "debug" ) + +func statsDebug(ctx context.Context, repo restic.Repository) error { + Warnf("Collecting size statistics\n\n") + for _, t := range []restic.FileType{restic.KeyFile, restic.LockFile, restic.IndexFile, restic.PackFile} { + hist, err := statsDebugFileType(ctx, repo, t) + if err != nil { + return err + } + Warnf("File Type: %v\n%v\n", t, hist) + } + + hist := statsDebugBlobs(ctx, repo) + for _, t := range []restic.BlobType{restic.DataBlob, restic.TreeBlob} { + Warnf("Blob Type: %v\n%v\n\n", t, hist[t]) + } + + return nil +} + +func statsDebugFileType(ctx context.Context, repo restic.Repository, tpe restic.FileType) (*sizeHistogram, error) { + hist := newSizeHistogram(2 * repository.MaxPackSize) + err := repo.List(ctx, tpe, func(id restic.ID, size int64) error { + hist.Add(uint64(size)) + return nil + }) + + return hist, err +} + +func statsDebugBlobs(ctx context.Context, repo restic.Repository) [restic.NumBlobTypes]*sizeHistogram { + var hist [restic.NumBlobTypes]*sizeHistogram + for i := 0; i < len(hist); i++ { + hist[i] = newSizeHistogram(2 * chunker.MaxSize) + } + + repo.Index().Each(ctx, func(pb restic.PackedBlob) { + hist[pb.Type].Add(uint64(pb.Length)) + }) + + return hist +} + +type sizeClass struct { + lower, upper uint64 + count int64 +} + +type sizeHistogram struct { + count int64 + totalSize uint64 + buckets []sizeClass + oversized []uint64 +} + +func newSizeHistogram(sizeLimit uint64) *sizeHistogram { + h := &sizeHistogram{} + h.buckets = append(h.buckets, sizeClass{0, 0, 0}) + + lowerBound := uint64(1) + growthFactor := uint64(10) + + for lowerBound < sizeLimit { + upperBound := lowerBound*growthFactor - 1 + if upperBound > sizeLimit { + upperBound = sizeLimit + } + h.buckets = append(h.buckets, sizeClass{lowerBound, upperBound, 0}) + lowerBound *= growthFactor + } + + return h +} + +func (s *sizeHistogram) Add(size uint64) { + s.count++ + s.totalSize += size + + for i, bucket := range s.buckets { + if size >= bucket.lower && size <= bucket.upper { + s.buckets[i].count++ + return + } + } + + s.oversized = append(s.oversized, size) +} + +func (s sizeHistogram) String() string { + var out strings.Builder + + out.WriteString(fmt.Sprintf("Count: %d\n", s.count)) + out.WriteString(fmt.Sprintf("Total Size: %s\n", ui.FormatBytes(s.totalSize))) + + t := table.New() + t.AddColumn("Size", "{{.SizeRange}}") + t.AddColumn("Count", "{{.Count}}") + type line struct { + SizeRange string + Count int64 + } + + // only print up to the highest used bucket size + lastFilledIdx := 0 + for i := 0; i < len(s.buckets); i++ { + if s.buckets[i].count != 0 { + lastFilledIdx = i + } + } + + var lines []line + hasStarted := false + for i, b := range s.buckets { + if i > lastFilledIdx { + break + } + + if b.count > 0 { + hasStarted = true + } + if hasStarted { + lines = append(lines, line{ + SizeRange: fmt.Sprintf("%d - %d Byte", b.lower, b.upper), + Count: b.count, + }) + } + } + longestRange := 0 + for _, l := range lines { + if longestRange < len(l.SizeRange) { + longestRange = len(l.SizeRange) + } + } + for i := range lines { + lines[i].SizeRange = strings.Repeat(" ", longestRange-len(lines[i].SizeRange)) + lines[i].SizeRange + t.AddRow(lines[i]) + } + + _ = t.Write(&out) + + if len(s.oversized) > 0 { + out.WriteString(fmt.Sprintf("Oversized: %v\n", s.oversized)) + } + return out.String() +} diff --git a/cmd/restic/cmd_stats_test.go b/cmd/restic/cmd_stats_test.go new file mode 100644 index 000000000..02d37acd9 --- /dev/null +++ b/cmd/restic/cmd_stats_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "testing" + + rtest "github.com/restic/restic/internal/test" +) + +func TestSizeHistogramNew(t *testing.T) { + h := newSizeHistogram(42) + + exp := &sizeHistogram{ + count: 0, + totalSize: 0, + buckets: []sizeClass{ + {0, 0, 0}, + {1, 9, 0}, + {10, 42, 0}, + }, + } + + rtest.Equals(t, exp, h) +} + +func TestSizeHistogramAdd(t *testing.T) { + h := newSizeHistogram(42) + for i := uint64(0); i < 45; i++ { + h.Add(i) + } + + exp := &sizeHistogram{ + count: 45, + totalSize: 990, + buckets: []sizeClass{ + {0, 0, 1}, + {1, 9, 9}, + {10, 42, 33}, + }, + oversized: []uint64{43, 44}, + } + + rtest.Equals(t, exp, h) +} + +func TestSizeHistogramString(t *testing.T) { + t.Run("overflow", func(t *testing.T) { + h := newSizeHistogram(42) + h.Add(8) + h.Add(50) + + rtest.Equals(t, "Count: 2\nTotal Size: 58 B\nSize Count\n-----------------\n1 - 9 Byte 1\n-----------------\nOversized: [50]\n", h.String()) + }) + + t.Run("withZero", func(t *testing.T) { + h := newSizeHistogram(42) + h.Add(0) + h.Add(1) + h.Add(10) + + rtest.Equals(t, "Count: 3\nTotal Size: 11 B\nSize Count\n-------------------\n 0 - 0 Byte 1\n 1 - 9 Byte 1\n10 - 42 Byte 1\n-------------------\n", h.String()) + }) +} From 0fcb6c7f9451a3f09d0a62b14b827948c49e149a Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 17 Jun 2023 17:47:25 +0200 Subject: [PATCH 69/90] ask for debug statistics in resource usage issues --- CONTRIBUTING.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e0f2e2c08..39a829337 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,6 +68,9 @@ it might be necessary to manually clean up stale lock files using On Windows, please set the environment variable `RESTIC_DEBUG_STACKTRACE_SIGINT` to `true` and press `Ctrl-C` to create a stacktrace. +If you think restic uses too much memory or a too large cache directory, then +please include the output of `restic stats --mode debug`. + Development Environment ======================= From 2293835242df548251244e02fcbf921cc19db461 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 19 May 2023 17:16:29 +0200 Subject: [PATCH 70/90] Release multi-platform docker containers --- docker/Dockerfile.release | 20 ++++++++++++++++++++ helpers/prepare-release/main.go | 24 +++++++++++++++--------- 2 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 docker/Dockerfile.release diff --git a/docker/Dockerfile.release b/docker/Dockerfile.release new file mode 100644 index 000000000..a2564b6d5 --- /dev/null +++ b/docker/Dockerfile.release @@ -0,0 +1,20 @@ +FROM --platform=$BUILDPLATFORM alpine:latest as helper + +ARG VERSION +ARG TARGETOS +ARG TARGETARCH + +# add release binary for the appropriate platform +COPY restic_${VERSION}_${TARGETOS}_${TARGETARCH}.bz2 / +RUN apk add --update --no-cache bzip2 +RUN set -e && \ + bzcat restic_${VERSION}_${TARGETOS}_${TARGETARCH}.bz2 > restic && \ + chmod +x restic + + +FROM alpine:latest + +COPY --from=helper /restic /usr/bin +RUN apk add --update --no-cache ca-certificates fuse openssh-client tzdata jq + +ENTRYPOINT ["/usr/bin/restic"] diff --git a/helpers/prepare-release/main.go b/helpers/prepare-release/main.go index 03924b0d9..2340a65d4 100644 --- a/helpers/prepare-release/main.go +++ b/helpers/prepare-release/main.go @@ -409,13 +409,19 @@ func signFiles(filenames ...string) { } } -func updateDocker(outputDir, version string) { - cmd := fmt.Sprintf("bzcat %s/restic_%s_linux_amd64.bz2 > restic", outputDir, version) - run("sh", "-c", cmd) - run("chmod", "+x", "restic") - run("docker", "pull", "alpine:latest") - run("docker", "build", "--rm", "--tag", "restic/restic:latest", "-f", "docker/Dockerfile", ".") - run("docker", "tag", "restic/restic:latest", "restic/restic:"+version) +func updateDocker(outputDir, version string) string { + run("docker", "buildx", "create", "--name", "restic-release-builder", "--driver", "docker-container", "--bootstrap") + + cmds := "" + + for _, tag := range []string{"restic/restic:latest", "restic/restic:" + version} { + cmd := fmt.Sprintf("docker buildx build --builder restic-release-builder --platform linux/386,linux/amd64,linux/arm,linux/arm64 --pull --tag %q -f docker/Dockerfile.release --build-arg VERSION=%q %q", tag, version, outputDir) + run("sh", "-c", cmd) + + cmds += cmd + " --push\n" + } + + return cmds + "\ndocker buildx rm restic-release-builder" } func tempdir(prefix string) string { @@ -470,9 +476,9 @@ func main() { signFiles(filepath.Join(opts.OutputDir, "SHA256SUMS"), tarFilename) - updateDocker(opts.OutputDir, opts.Version) + dockerCmds := updateDocker(opts.OutputDir, opts.Version) msg("done, output dir is %v", opts.OutputDir) - msg("now run:\n\ngit push --tags origin master\ndocker push restic/restic:latest\ndocker push restic/restic:%s\n", opts.Version) + msg("now run:\n\ngit push --tags origin master\n%s\n", dockerCmds) } From 43fa0515462f626084859acf6300ce8c3a0653f7 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 19 May 2023 17:43:11 +0200 Subject: [PATCH 71/90] Directly build restic binary in release Docker container --- .dockerignore | 1 - docker/Dockerfile.release | 16 +++++++--------- helpers/build-release-binaries/main.go | 10 +++++++++- helpers/prepare-release/main.go | 24 ++++++++++++------------ 4 files changed, 28 insertions(+), 23 deletions(-) diff --git a/.dockerignore b/.dockerignore index 2e1b785e0..b7f28c69f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,7 +4,6 @@ changelog/ doc/ docker/ -helpers/ # Files .gitignore diff --git a/docker/Dockerfile.release b/docker/Dockerfile.release index a2564b6d5..01f9df150 100644 --- a/docker/Dockerfile.release +++ b/docker/Dockerfile.release @@ -1,20 +1,18 @@ -FROM --platform=$BUILDPLATFORM alpine:latest as helper +# the official binaries are cross-built from Linux running on an AMD64 host +# other architectures also seem to generate identical binaries but stay on the safe side +FROM --platform=linux/amd64 restic/builder:latest as helper -ARG VERSION ARG TARGETOS ARG TARGETARCH -# add release binary for the appropriate platform -COPY restic_${VERSION}_${TARGETOS}_${TARGETARCH}.bz2 / -RUN apk add --update --no-cache bzip2 -RUN set -e && \ - bzcat restic_${VERSION}_${TARGETOS}_${TARGETARCH}.bz2 > restic && \ - chmod +x restic +COPY . /restic +RUN go run helpers/build-release-binaries/main.go --platform $TARGETOS/$TARGETARCH --skip-compress +RUN mv /output/restic_${TARGETOS}_${TARGETARCH} /output/restic FROM alpine:latest -COPY --from=helper /restic /usr/bin +COPY --from=helper /output/restic /usr/bin RUN apk add --update --no-cache ca-certificates fuse openssh-client tzdata jq ENTRYPOINT ["/usr/bin/restic"] diff --git a/helpers/build-release-binaries/main.go b/helpers/build-release-binaries/main.go index 6938aff84..caa90ff82 100644 --- a/helpers/build-release-binaries/main.go +++ b/helpers/build-release-binaries/main.go @@ -22,6 +22,8 @@ var opts = struct { OutputDir string Tags string PlatformSubset string + Platform string + SkipCompress bool Version string }{} @@ -31,6 +33,8 @@ func init() { pflag.StringVarP(&opts.OutputDir, "output", "o", "/output", "path to the output `directory`") pflag.StringVar(&opts.Tags, "tags", "", "additional build `tags`") pflag.StringVar(&opts.PlatformSubset, "platform-subset", "", "specify `n/t` to only build this subset") + pflag.StringVarP(&opts.Platform, "platform", "p", "", "specify `os/arch` to only build this specific platform") + pflag.BoolVar(&opts.SkipCompress, "skip-compress", false, "skip binary compression step") pflag.StringVar(&opts.Version, "version", "", "use `x.y.z` as the version for output files") pflag.Parse() } @@ -188,7 +192,9 @@ func buildForTarget(sourceDir, outputDir, goos, goarch string) (filename string) filename = build(sourceDir, outputDir, goos, goarch) touch(filepath.Join(outputDir, filename), mtime) chmod(filepath.Join(outputDir, filename), 0755) - filename = compress(goos, outputDir, filename) + if !opts.SkipCompress { + filename = compress(goos, outputDir, filename) + } return filename } @@ -311,6 +317,8 @@ func main() { if err != nil { die("%s", err) } + } else if opts.Platform != "" { + targets = buildPlatformList([]string{opts.Platform}) } sourceDir := abs(opts.SourceDir) diff --git a/helpers/prepare-release/main.go b/helpers/prepare-release/main.go index 2340a65d4..a6c7bd4f4 100644 --- a/helpers/prepare-release/main.go +++ b/helpers/prepare-release/main.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "fmt" + "math/rand" "os" "os/exec" "path/filepath" @@ -409,19 +410,19 @@ func signFiles(filenames ...string) { } } -func updateDocker(outputDir, version string) string { - run("docker", "buildx", "create", "--name", "restic-release-builder", "--driver", "docker-container", "--bootstrap") +func updateDocker(sourceDir, version string) string { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + builderName := fmt.Sprintf("restic-release-builder-%d", r.Int()) + run("docker", "buildx", "create", "--name", builderName, "--driver", "docker-container", "--bootstrap") - cmds := "" + buildCmd := fmt.Sprintf("docker buildx build --builder %s --platform linux/386,linux/amd64,linux/arm,linux/arm64 --pull -f docker/Dockerfile.release %q", builderName, sourceDir) + run("sh", "-c", buildCmd+" --no-cache") + publishCmds := "" for _, tag := range []string{"restic/restic:latest", "restic/restic:" + version} { - cmd := fmt.Sprintf("docker buildx build --builder restic-release-builder --platform linux/386,linux/amd64,linux/arm,linux/arm64 --pull --tag %q -f docker/Dockerfile.release --build-arg VERSION=%q %q", tag, version, outputDir) - run("sh", "-c", cmd) - - cmds += cmd + " --push\n" + publishCmds += buildCmd + fmt.Sprintf(" --tag %q --push\n", tag) } - - return cmds + "\ndocker buildx rm restic-release-builder" + return publishCmds + "\ndocker buildx rm " + builderName } func tempdir(prefix string) string { @@ -470,15 +471,14 @@ func main() { extractTar(tarFilename, sourceDir) runBuild(sourceDir, opts.OutputDir, opts.Version) - rmdir(sourceDir) sha256sums(opts.OutputDir, filepath.Join(opts.OutputDir, "SHA256SUMS")) signFiles(filepath.Join(opts.OutputDir, "SHA256SUMS"), tarFilename) - dockerCmds := updateDocker(opts.OutputDir, opts.Version) + dockerCmds := updateDocker(sourceDir, opts.Version) msg("done, output dir is %v", opts.OutputDir) - msg("now run:\n\ngit push --tags origin master\n%s\n", dockerCmds) + msg("now run:\n\ngit push --tags origin master\n%s\n\nrm -rf %q", dockerCmds, sourceDir) } From eff3124f1599310bb41718208b761af06ee4209c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 May 2023 11:10:00 +0200 Subject: [PATCH 72/90] CI: Setup automatic container builds for ghcr.io Containers are built for new tags and pushes to the master branch. --- .github/workflows/docker.yml | 58 ++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 000000000..9fe5b3a30 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,58 @@ + +name: Create and publish a Docker image + +on: + push: + tags: + - 'v*' + branches: + - 'master' + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Set up QEMU + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c + + - name: Build and push Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + push: true + context: . + file: docker/Dockerfile.release + platforms: linux/386,linux/amd64,linux/arm,linux/arm64 + pull: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} From b0987ff570317cee25b1f2cc869fcdcf1d379490 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 May 2023 12:12:50 +0200 Subject: [PATCH 73/90] CI: only build containers on restic/restic --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9fe5b3a30..43c427109 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -14,6 +14,7 @@ env: jobs: build-and-push-image: + if: github.repository == 'restic/restic' runs-on: ubuntu-latest permissions: contents: read From ccd19b7e88e9c3bf2bb5889e46db6135d393ee05 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 May 2023 12:23:45 +0200 Subject: [PATCH 74/90] CI: run cloud backend tests only on restic/restic --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9bca02f8d..8e8a5b099 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -187,7 +187,7 @@ jobs: # own repo, otherwise the secrets are not available # Skip for Dependabot pull requests as these are run without secrets # https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#responding-to-events - if: (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && (github.actor != 'dependabot[bot]') && matrix.test_cloud_backends + if: ((github.repository == 'restic/restic' && github.event_name == 'push') || github.event.pull_request.head.repo.full_name == github.repository) && (github.actor != 'dependabot[bot]') && matrix.test_cloud_backends - name: Check changelog files with calens run: | From c181b51360de7e0754663bb94cc3b9f54080a5e4 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 9 Jun 2023 12:40:25 +0200 Subject: [PATCH 75/90] doc: Update docker build process --- doc/020_installation.rst | 6 ++++++ doc/developer_information.rst | 2 ++ 2 files changed, 8 insertions(+) diff --git a/doc/020_installation.rst b/doc/020_installation.rst index b53c350b1..a39ae91e9 100644 --- a/doc/020_installation.rst +++ b/doc/020_installation.rst @@ -265,6 +265,12 @@ binary, you can get it with `docker pull` like this: $ docker pull restic/restic +The container is also available on the GitHub Container Registry: + +.. code-block:: console + + $ docker pull ghcr.io/restic/restic + Restic relies on the hostname for various operations. Make sure to set a static hostname using `--hostname` when creating a Docker container, otherwise Docker will assign a random hostname each time. diff --git a/doc/developer_information.rst b/doc/developer_information.rst index 307851757..9de517901 100644 --- a/doc/developer_information.rst +++ b/doc/developer_information.rst @@ -127,3 +127,5 @@ required argument is the new version number (in `Semantic Versioning go run helpers/prepare-release/main.go 0.14.0 Checks can be skipped on demand via flags, please see ``--help`` for details. + +The build process requires ``docker``, ``docker-buildx`` and ``qemu-user-static-binfmt``. From 31e07cecbb75370aa055bd5a81cda7945a49222c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 9 Jun 2023 12:40:49 +0200 Subject: [PATCH 76/90] docker: update to Go 1.20 for custom container builds --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 72fc85093..ecc283f8a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19-alpine AS builder +FROM golang:1.20-alpine AS builder WORKDIR /go/src/github.com/restic/restic From 6b82cce1bd2c5d2c20566749a7229a538552731f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 9 Jun 2023 12:46:29 +0200 Subject: [PATCH 77/90] add changelog for multiplatform containers --- changelog/unreleased/issue-2359 | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 changelog/unreleased/issue-2359 diff --git a/changelog/unreleased/issue-2359 b/changelog/unreleased/issue-2359 new file mode 100644 index 000000000..0399a96f1 --- /dev/null +++ b/changelog/unreleased/issue-2359 @@ -0,0 +1,11 @@ +Enhancement: Provide multi-platform Docker containers + +The official Docker containers are now built for the architectures linux/386, +linux/amd64, linux/arm and linux/arm64. + +As an alternative to the Docker Hub, the Docker containers are now also +available on ghcr.io, the GitHub Container Registry. + +https://github.com/restic/restic/issues/2359 +https://github.com/restic/restic/issues/4269 +https://github.com/restic/restic/pull/4364 From 3a93e28605ae6b64f0e854fd4601ac814517cb94 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 17 Jun 2023 16:22:50 +0200 Subject: [PATCH 78/90] CI: Remove .dockerignore to ensure reproducible builds Since go 1.18, built binaries also include VCS information such as the built commit. This information is also included in the official binaries. To ensure that the Docker container recreates the same binaries, the .git folder must also be transferred into the container. Thus, remove the .dockerignore file. The copied files must also be owned by the current user within the container, as git refuses to work otherwise. --- .dockerignore | 11 ----------- docker/Dockerfile.release | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) delete mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index b7f28c69f..000000000 --- a/.dockerignore +++ /dev/null @@ -1,11 +0,0 @@ -# Folders -.git/ -.github/ -changelog/ -doc/ -docker/ - -# Files -.gitignore -.golangci.yml -*.md diff --git a/docker/Dockerfile.release b/docker/Dockerfile.release index 01f9df150..ccf80376a 100644 --- a/docker/Dockerfile.release +++ b/docker/Dockerfile.release @@ -5,7 +5,7 @@ FROM --platform=linux/amd64 restic/builder:latest as helper ARG TARGETOS ARG TARGETARCH -COPY . /restic +COPY --chown=build . /restic RUN go run helpers/build-release-binaries/main.go --platform $TARGETOS/$TARGETARCH --skip-compress RUN mv /output/restic_${TARGETOS}_${TARGETARCH} /output/restic From 39299e36efe3de59db01bbc60f101adbf6efd6b7 Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Fri, 15 May 2020 21:40:09 -0700 Subject: [PATCH 79/90] add json documentation --- doc/075_scripting.rst | 249 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 249 insertions(+) diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index dc7f782dc..61c1949da 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -37,3 +37,252 @@ exit code if a different error is encountered (e.g.: incorrect password to ``cat config``) and it may print a different error message. If there are no errors, restic will return a zero exit code and print the repository metadata. + +Restic and json +*************** + +Restic can output json data if requested with the ``--json`` flag. +The structure of that data varies depending on the circumstance. The +json output of Most restic commands are documented here. + +.. note:: + Not all commands support json output. If a command does not support json output, + at the time of writing, it is not supported yet. (feel free to submit a pull request!) + +Backup +------ + +backup has multiple json structures, outlined below. + +Status +^^^^^^ + ++----------------------+---------------------------------------------------------+ +|``message_type`` | always "status" | ++----------------------+---------------------------------------------------------+ +|``seconds_elapsed`` | Time since backup started | ++----------------------+---------------------------------------------------------+ +|``seconds_remaining`` | Estimated time remaining | ++----------------------+---------------------------------------------------------+ +|``percent_done`` | Percentage of data backed up. (bytes_done/total_bytes) | ++----------------------+---------------------------------------------------------+ +|``total_files`` | Total number of files detected | ++----------------------+---------------------------------------------------------+ +|``files_done`` | Files completed (backed up or skipped) | ++----------------------+---------------------------------------------------------+ +|``total_bytes`` | Total number of bytes in backup set | ++----------------------+---------------------------------------------------------+ +|``bytes_done`` | Number of bytes completed | ++----------------------+---------------------------------------------------------+ +|``error_count`` | Number of errors | ++----------------------+---------------------------------------------------------+ +|``current_files`` | List of files currently being backed up | ++----------------------+---------------------------------------------------------+ + +Error +^^^^^ + ++----------------------+--------------------------------+ +| ``message_type`` | always "error" | ++----------------------+--------------------------------+ +| ``error`` | error message | ++----------------------+--------------------------------+ +| ``during`` | what restic was trying to do | ++----------------------+--------------------------------+ +| ``item`` | what item was being processed | ++----------------------+--------------------------------+ + +Verbose Status +^^^^^^^^^^^^^^ + ++----------------------+-------------------------------------------+ +| ``message_type`` | Always "verbose_status" | ++----------------------+-------------------------------------------+ +| ``action`` | Either "new", "unchanged" or "modified" | ++----------------------+-------------------------------------------+ +| ``item`` | The item in question | ++----------------------+-------------------------------------------+ +| ``duration`` | How long it took, in seconds | ++----------------------+-------------------------------------------+ +| ``data_size`` | How big item is | ++----------------------+-------------------------------------------+ +| ``metadata_size`` | How big the metadata is | ++----------------------+-------------------------------------------+ +| ``total_files`` | how many total files there are. | ++----------------------+-------------------------------------------+ + +Summary +^^^^^^^ + ++---------------------------+---------------------------------------------------------+ +| ``message_type`` | Always "summary" | ++---------------------------+---------------------------------------------------------+ +| ``files_new`` | Number of new files | ++---------------------------+---------------------------------------------------------+ +| ``files_changed`` | Number of files that changed | ++---------------------------+---------------------------------------------------------+ +| ``files_unmodified`` | Number of files that did not change | ++---------------------------+---------------------------------------------------------+ +| ``dirs_new`` | Number of new directories | ++---------------------------+---------------------------------------------------------+ +| ``dirs_changed`` | Number of directories that changed | ++---------------------------+---------------------------------------------------------+ +| ``dirs_unmodified`` | Number of directories that did not change | ++---------------------------+---------------------------------------------------------+ +| ``data_blobs`` | Number of data blobs | ++---------------------------+---------------------------------------------------------+ +| ``tree_blobs`` | Number of tree blobs | ++---------------------------+---------------------------------------------------------+ +| ``data_added`` | Amount of data added, in bytes | ++---------------------------+---------------------------------------------------------+ +| ``total_files_processed`` | Total number of files processed | ++---------------------------+---------------------------------------------------------+ +| ``total_bytes_processed`` | Total number of bytes processed | ++---------------------------+---------------------------------------------------------+ +| ``total_duration`` | Total time it took for the operation to complete | ++---------------------------+---------------------------------------------------------+ +| ``snapshot_id`` | the ID of the new snapshot | ++---------------------------+---------------------------------------------------------+ + +snapshots +--------- + +Snapshots returns a single json structure with a number of optional fields. + ++----------------+------------------------------------------------------------------------+ +| ``hostname`` | contains the hostname of the machine that's being backed up. | ++----------------+------------------------------------------------------------------------+ +| ``username`` | contains the username that the backup command was run as. | ++----------------+------------------------------------------------------------------------+ +| ``excludes`` | contains a list of paths and globs that were excluded from the backup. | ++----------------+------------------------------------------------------------------------+ +| ``tags`` | contains a list of tags for the snapshot in question. | ++----------------+------------------------------------------------------------------------+ +| ``id`` | contains the long snapshot id. | ++----------------+------------------------------------------------------------------------+ +| ``short_id`` | contains the short snapshot id. | ++----------------+------------------------------------------------------------------------+ +| ``time`` | contains the timestamp of the backup. | ++----------------+------------------------------------------------------------------------+ +| ``parent`` | contains the id of the previous backup. | ++----------------+------------------------------------------------------------------------+ +| ``tree`` | contains something... | ++----------------+------------------------------------------------------------------------+ +| ``paths`` | contains a list of paths that were included in the backup. | ++----------------+------------------------------------------------------------------------+ + +cat +--- + +Cat will return data about various objects in the repository, already in json form. +By specifying ``--json``, it will suppress any non-json messages the command generates. + +find +---- + + ++-----------------+------------------------------------------+ +| ``path`` | Object path | ++-----------------+------------------------------------------+ +| ``permissions`` | unix permissions | ++-----------------+------------------------------------------+ +| ``type`` | what type it is e.g. file, dir, etc... | ++-----------------+------------------------------------------+ +| ``atime`` | Access time | ++-----------------+------------------------------------------+ +| ``mtime`` | Modification time | ++-----------------+------------------------------------------+ +| ``ctime`` | Creation time | ++-----------------+------------------------------------------+ +| ``name`` | Object name | ++-----------------+------------------------------------------+ +| ``user`` | Name of owner | ++-----------------+------------------------------------------+ +| ``group`` | Name of group | ++-----------------+------------------------------------------+ +| ``uid`` | ID of owner | ++-----------------+------------------------------------------+ +| ``gid`` | ID of group | ++-----------------+------------------------------------------+ +| ``size`` | size of object in bytes | ++-----------------+------------------------------------------+ + +key list +-------- + ++--------------+------------------------------------+ +| ``current`` | Is currently used key? | ++--------------+------------------------------------+ +| ``id`` | Unique key ID | ++--------------+------------------------------------+ +| ``userName`` | user who created it | ++--------------+------------------------------------+ +| ``hostName`` | name of machine it was created on | ++--------------+------------------------------------+ +| ``created`` | timestamp when it was created | ++--------------+------------------------------------+ + +ls +-- + +snapshot +^^^^^^^^ + ++-----------------+-------------------------------------+ +| ``time`` | Snapshot time | ++-----------------+-------------------------------------+ +| ``tree`` | Snapshot tree root | ++-----------------+-------------------------------------+ +| ``paths`` | List of paths included in snapshot | ++-----------------+-------------------------------------+ +| ``hostname`` | hostname of snapshot | ++-----------------+-------------------------------------+ +| ``username`` | user snapshot was run as | ++-----------------+-------------------------------------+ +| ``uid`` | uid of backup process | ++-----------------+-------------------------------------+ +| ``gid`` | gid of backup process | ++-----------------+-------------------------------------+ +| ``id`` | snapshot id, long form | ++-----------------+-------------------------------------+ +| ``short_id`` | snapshot id, short form | ++-----------------+-------------------------------------+ +| ``struct_type`` | always "snapshot" | ++-----------------+-------------------------------------+ + + +node +^^^^ + ++-----------------+--------------------------+ +| ``name`` | node name | ++-----------------+--------------------------+ +| ``type`` | node type | ++-----------------+--------------------------+ +| ``path`` | node path | ++-----------------+--------------------------+ +| ``uid`` | uid of node | ++-----------------+--------------------------+ +| ``gid`` | gid of node | ++-----------------+--------------------------+ +| ``size`` | size in bytes | ++-----------------+--------------------------+ +| ``mode`` | node mode | ++-----------------+--------------------------+ +| ``atime`` | node access time | ++-----------------+--------------------------+ +| ``mtime`` | node modification time | ++-----------------+--------------------------+ +| ``ctime`` | node creation time | ++-----------------+--------------------------+ +| ``struct_type`` | always "node" | ++-----------------+--------------------------+ + +stats +----- + ++----------------------+---------------------------------------------+ +| ``total_size`` | Repository size in bytes | ++----------------------+---------------------------------------------+ +| ``total_file_count`` | Number of files backed up in the repository | ++----------------------+---------------------------------------------+ From e457fe22bc68224600d41efe1e28843b37289b5d Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Wed, 25 Nov 2020 23:54:53 -0800 Subject: [PATCH 80/90] Documentation improvement --- doc/075_scripting.rst | 221 ++++++++++++++++++++++++------------------ 1 file changed, 127 insertions(+), 94 deletions(-) diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index 61c1949da..ec36afe43 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -38,63 +38,68 @@ to ``cat config``) and it may print a different error message. If there are no errors, restic will return a zero exit code and print the repository metadata. -Restic and json +Restic and JSON *************** Restic can output json data if requested with the ``--json`` flag. The structure of that data varies depending on the circumstance. The -json output of Most restic commands are documented here. +json output of most restic commands are documented here. .. note:: Not all commands support json output. If a command does not support json output, - at the time of writing, it is not supported yet. (feel free to submit a pull request!) + feel free to submit a pull request! Backup ------ -backup has multiple json structures, outlined below. +The backup command has multiple json structures, outlined below. + +During the backup process, Restic will print out a stream of new-line separated JSON +messages. You can determine the nature of the message by the ``message_type`` field. Status ^^^^^^ -+----------------------+---------------------------------------------------------+ -|``message_type`` | always "status" | -+----------------------+---------------------------------------------------------+ -|``seconds_elapsed`` | Time since backup started | -+----------------------+---------------------------------------------------------+ -|``seconds_remaining`` | Estimated time remaining | -+----------------------+---------------------------------------------------------+ -|``percent_done`` | Percentage of data backed up. (bytes_done/total_bytes) | -+----------------------+---------------------------------------------------------+ -|``total_files`` | Total number of files detected | -+----------------------+---------------------------------------------------------+ -|``files_done`` | Files completed (backed up or skipped) | -+----------------------+---------------------------------------------------------+ -|``total_bytes`` | Total number of bytes in backup set | -+----------------------+---------------------------------------------------------+ -|``bytes_done`` | Number of bytes completed | -+----------------------+---------------------------------------------------------+ -|``error_count`` | Number of errors | -+----------------------+---------------------------------------------------------+ -|``current_files`` | List of files currently being backed up | -+----------------------+---------------------------------------------------------+ ++----------------------+------------------------------------------------------------+ +|``message_type`` | Always "status" | ++----------------------+------------------------------------------------------------+ +|``seconds_elapsed`` | Time since backup started | ++----------------------+------------------------------------------------------------+ +|``seconds_remaining`` | Estimated time remaining | ++----------------------+------------------------------------------------------------+ +|``percent_done`` | Percentage of data backed up (bytes_done/total_bytes) | ++----------------------+------------------------------------------------------------+ +|``total_files`` | Total number of files detected | ++----------------------+------------------------------------------------------------+ +|``files_done`` | Files completed (backed up or confirmed in repo) | ++----------------------+------------------------------------------------------------+ +|``total_bytes`` | Total number of bytes in backup set | ++----------------------+------------------------------------------------------------+ +|``bytes_done`` | Number of bytes completed (backed up or confirmed in repo) | ++----------------------+------------------------------------------------------------+ +|``error_count`` | Number of errors | ++----------------------+------------------------------------------------------------+ +|``current_files`` | List of files currently being backed up | ++----------------------+------------------------------------------------------------+ Error ^^^^^ -+----------------------+--------------------------------+ -| ``message_type`` | always "error" | -+----------------------+--------------------------------+ -| ``error`` | error message | -+----------------------+--------------------------------+ -| ``during`` | what restic was trying to do | -+----------------------+--------------------------------+ -| ``item`` | what item was being processed | -+----------------------+--------------------------------+ ++----------------------+-------------------------------------------+ +| ``message_type`` | Always "error" | ++----------------------+-------------------------------------------+ +| ``error`` | Error message | ++----------------------+-------------------------------------------+ +| ``during`` | What restic was trying to do | ++----------------------+-------------------------------------------+ +| ``item`` | Usually, the path of the problematic file | ++----------------------+-------------------------------------------+ Verbose Status ^^^^^^^^^^^^^^ +Verbose status is a status line that supplements + +----------------------+-------------------------------------------+ | ``message_type`` | Always "verbose_status" | +----------------------+-------------------------------------------+ @@ -104,16 +109,18 @@ Verbose Status +----------------------+-------------------------------------------+ | ``duration`` | How long it took, in seconds | +----------------------+-------------------------------------------+ -| ``data_size`` | How big item is | +| ``data_size`` | How big the item is | +----------------------+-------------------------------------------+ | ``metadata_size`` | How big the metadata is | +----------------------+-------------------------------------------+ -| ``total_files`` | how many total files there are. | +| ``total_files`` | Total number of files | +----------------------+-------------------------------------------+ Summary ^^^^^^^ +Summary is the last output line in a successful backup. + +---------------------------+---------------------------------------------------------+ | ``message_type`` | Always "summary" | +---------------------------+---------------------------------------------------------+ @@ -141,34 +148,34 @@ Summary +---------------------------+---------------------------------------------------------+ | ``total_duration`` | Total time it took for the operation to complete | +---------------------------+---------------------------------------------------------+ -| ``snapshot_id`` | the ID of the new snapshot | +| ``snapshot_id`` | The short ID of the new snapshot | +---------------------------+---------------------------------------------------------+ snapshots --------- -Snapshots returns a single json structure with a number of optional fields. +The snapshots command returns a single JSON object, an array with the structure outlined below. +----------------+------------------------------------------------------------------------+ -| ``hostname`` | contains the hostname of the machine that's being backed up. | +| ``hostname`` | The hostname of the machine that's being backed up | +----------------+------------------------------------------------------------------------+ -| ``username`` | contains the username that the backup command was run as. | +| ``username`` | The username that the backup command was run as | +----------------+------------------------------------------------------------------------+ -| ``excludes`` | contains a list of paths and globs that were excluded from the backup. | +| ``excludes`` | A list of paths and globs that were excluded from the backup | +----------------+------------------------------------------------------------------------+ -| ``tags`` | contains a list of tags for the snapshot in question. | +| ``tags`` | A list of tags for the snapshot in question | +----------------+------------------------------------------------------------------------+ -| ``id`` | contains the long snapshot id. | +| ``id`` | The long snapshot ID | +----------------+------------------------------------------------------------------------+ -| ``short_id`` | contains the short snapshot id. | +| ``short_id`` | The short snapshot ID | +----------------+------------------------------------------------------------------------+ -| ``time`` | contains the timestamp of the backup. | +| ``time`` | The timestamp of when the backup was started | +----------------+------------------------------------------------------------------------+ -| ``parent`` | contains the id of the previous backup. | +| ``parent`` | The ID of the previous snapshot | +----------------+------------------------------------------------------------------------+ -| ``tree`` | contains something... | +| ``tree`` | The ID of the root tree blob | +----------------+------------------------------------------------------------------------+ -| ``paths`` | contains a list of paths that were included in the backup. | +| ``paths`` | A list of paths that were included in the backup | +----------------+------------------------------------------------------------------------+ cat @@ -180,51 +187,75 @@ By specifying ``--json``, it will suppress any non-json messages the command gen find ---- +The find command outputs an array of json objects with matches for your search term. These +matches are organized by snapshot. -+-----------------+------------------------------------------+ -| ``path`` | Object path | -+-----------------+------------------------------------------+ -| ``permissions`` | unix permissions | -+-----------------+------------------------------------------+ -| ``type`` | what type it is e.g. file, dir, etc... | -+-----------------+------------------------------------------+ -| ``atime`` | Access time | -+-----------------+------------------------------------------+ -| ``mtime`` | Modification time | -+-----------------+------------------------------------------+ -| ``ctime`` | Creation time | -+-----------------+------------------------------------------+ -| ``name`` | Object name | -+-----------------+------------------------------------------+ -| ``user`` | Name of owner | -+-----------------+------------------------------------------+ -| ``group`` | Name of group | -+-----------------+------------------------------------------+ -| ``uid`` | ID of owner | -+-----------------+------------------------------------------+ -| ``gid`` | ID of group | -+-----------------+------------------------------------------+ -| ``size`` | size of object in bytes | -+-----------------+------------------------------------------+ +Snapshot +^^^^^^^^ + ++-----------------+----------------------------------------------+ +| ``hits`` | The number of matches in the snapshot | ++-----------------+----------------------------------------------+ +| ``snapshot`` | The long ID of the snapshot | ++-----------------+----------------------------------------------+ +| ``matches`` | Array of JSON objects detailing a match. | ++-----------------+----------------------------------------------+ + + +Match +^^^^^ + ++-----------------+----------------------------------------------+ +| ``path`` | Object path | ++-----------------+----------------------------------------------+ +| ``permissions`` | UNIX permissions | ++-----------------+----------------------------------------------+ +| ``type`` | what type it is e.g. file, dir, etc... | ++-----------------+----------------------------------------------+ +| ``atime`` | Access time | ++-----------------+----------------------------------------------+ +| ``mtime`` | Modification time | ++-----------------+----------------------------------------------+ +| ``ctime`` | Change time | ++-----------------+----------------------------------------------+ +| ``name`` | Object name | ++-----------------+----------------------------------------------+ +| ``user`` | Name of owner | ++-----------------+----------------------------------------------+ +| ``group`` | Name of group | ++-----------------+----------------------------------------------+ +| ``mode`` | UNIX file mode, shorthand of ``permissions`` | ++-----------------+----------------------------------------------+ +| ``uid`` | ID of owner | ++-----------------+----------------------------------------------+ +| ``gid`` | ID of group | ++-----------------+----------------------------------------------+ +| ``size`` | Size of object in bytes | ++-----------------+----------------------------------------------+ key list -------- +The key list command returns an array of objects with the following structure. + +--------------+------------------------------------+ | ``current`` | Is currently used key? | +--------------+------------------------------------+ | ``id`` | Unique key ID | +--------------+------------------------------------+ -| ``userName`` | user who created it | +| ``userName`` | User who created it | +--------------+------------------------------------+ -| ``hostName`` | name of machine it was created on | +| ``hostName`` | Name of machine it was created on | +--------------+------------------------------------+ -| ``created`` | timestamp when it was created | +| ``created`` | Timestamp when it was created | +--------------+------------------------------------+ ls -- +The ls command spits out a series of newline-separated JSON objects, +the nature of which can be determined by the ``struct_type`` field. + snapshot ^^^^^^^^ @@ -235,19 +266,19 @@ snapshot +-----------------+-------------------------------------+ | ``paths`` | List of paths included in snapshot | +-----------------+-------------------------------------+ -| ``hostname`` | hostname of snapshot | +| ``hostname`` | Hostname of snapshot | +-----------------+-------------------------------------+ -| ``username`` | user snapshot was run as | +| ``username`` | User snapshot was run as | +-----------------+-------------------------------------+ -| ``uid`` | uid of backup process | +| ``uid`` | UID of backup process | +-----------------+-------------------------------------+ -| ``gid`` | gid of backup process | +| ``gid`` | GID of backup process | +-----------------+-------------------------------------+ -| ``id`` | snapshot id, long form | +| ``id`` | Snapshot ID, long form | +-----------------+-------------------------------------+ -| ``short_id`` | snapshot id, short form | +| ``short_id`` | Snapshot ID, short form | +-----------------+-------------------------------------+ -| ``struct_type`` | always "snapshot" | +| ``struct_type`` | Always "snapshot" | +-----------------+-------------------------------------+ @@ -255,27 +286,27 @@ node ^^^^ +-----------------+--------------------------+ -| ``name`` | node name | +| ``name`` | Node name | +-----------------+--------------------------+ -| ``type`` | node type | +| ``type`` | Node type | +-----------------+--------------------------+ -| ``path`` | node path | +| ``path`` | Node path | +-----------------+--------------------------+ -| ``uid`` | uid of node | +| ``uid`` | UID of node | +-----------------+--------------------------+ -| ``gid`` | gid of node | +| ``gid`` | GID of node | +-----------------+--------------------------+ -| ``size`` | size in bytes | +| ``size`` | Size in bytes | +-----------------+--------------------------+ -| ``mode`` | node mode | +| ``mode`` | Node mode | +-----------------+--------------------------+ -| ``atime`` | node access time | +| ``atime`` | Node access time | +-----------------+--------------------------+ -| ``mtime`` | node modification time | +| ``mtime`` | Node modification time | +-----------------+--------------------------+ -| ``ctime`` | node creation time | +| ``ctime`` | Node creation time | +-----------------+--------------------------+ -| ``struct_type`` | always "node" | +| ``struct_type`` | Always "node" | +-----------------+--------------------------+ stats @@ -286,3 +317,5 @@ stats +----------------------+---------------------------------------------+ | ``total_file_count`` | Number of files backed up in the repository | +----------------------+---------------------------------------------+ +| ``total_blob_count`` | Number of blobs in the repository | ++----------------------+---------------------------------------------+ From fb1170c1d6c996d36d5176d27fa3c3198ad5c4bb Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Thu, 26 Nov 2020 00:21:08 -0800 Subject: [PATCH 81/90] doc tweaks --- doc/075_scripting.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index ec36afe43..0fada390d 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -226,6 +226,10 @@ Match +-----------------+----------------------------------------------+ | ``mode`` | UNIX file mode, shorthand of ``permissions`` | +-----------------+----------------------------------------------+ +| ``device_id`` | Unique machine Identifier | ++-----------------+----------------------------------------------+ +| ``links`` | Number of hardlinks | ++-----------------+----------------------------------------------+ | ``uid`` | ID of owner | +-----------------+----------------------------------------------+ | ``gid`` | ID of group | @@ -270,9 +274,9 @@ snapshot +-----------------+-------------------------------------+ | ``username`` | User snapshot was run as | +-----------------+-------------------------------------+ -| ``uid`` | UID of backup process | +| ``uid`` | ID of owner | +-----------------+-------------------------------------+ -| ``gid`` | GID of backup process | +| ``gid`` | ID of group | +-----------------+-------------------------------------+ | ``id`` | Snapshot ID, long form | +-----------------+-------------------------------------+ From 1ce839228e39aa29f95e28fcc371b9f72f193311 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 9 Jun 2023 00:46:55 +0200 Subject: [PATCH 82/90] doc: Expand JSON documentation --- doc/075_scripting.rst | 414 +++++++++++++++++++++++++++++++----------- 1 file changed, 308 insertions(+), 106 deletions(-) diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index 0fada390d..9855b9ffb 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -38,24 +38,50 @@ to ``cat config``) and it may print a different error message. If there are no errors, restic will return a zero exit code and print the repository metadata. -Restic and JSON -*************** +JSON output +*********** -Restic can output json data if requested with the ``--json`` flag. +Restic outputs JSON data to ``stdout`` if requested with the ``--json`` flag. The structure of that data varies depending on the circumstance. The -json output of most restic commands are documented here. +JSON output of most restic commands are documented here. .. note:: - Not all commands support json output. If a command does not support json output, + Not all commands support JSON output. If a command does not support JSON output, feel free to submit a pull request! -Backup +.. warning:: + We try to keep the JSON output backwards compatible. However, new message types + or fields may be added at any time. Similarly, enum-like fields for which a fixed + list of allowed values is documented may be extended at any time. + + +Output formats +-------------- + +Currently only the output on ``stdout`` is JSON formatted. Errors printed on ``stderr`` +are still printed as plain text messages. The generated JSON output uses one of the +following two formats. + +Single JSON document +^^^^^^^^^^^^^^^^^^^^ + +Several commands output a single JSON document that can be parsed in its entirety. +Depending on the command, the output consists of either a single or multiple lines. + +JSON lines +^^^^^^^^^^ + +Several commands, in particular long running ones or those that generate a large output, +use a format also known as JSON lines. It consists of a stream of new-line separated JSON +messages. You can determine the nature of the message using the ``message_type`` field. + +As an exception, the ``ls`` command uses the field ``struct_type`` instead. + + +backup ------ -The backup command has multiple json structures, outlined below. - -During the backup process, Restic will print out a stream of new-line separated JSON -messages. You can determine the nature of the message by the ``message_type`` field. +The ``backup`` command uses the JSON lines format with the following message types. Status ^^^^^^ @@ -71,11 +97,11 @@ Status +----------------------+------------------------------------------------------------+ |``total_files`` | Total number of files detected | +----------------------+------------------------------------------------------------+ -|``files_done`` | Files completed (backed up or confirmed in repo) | +|``files_done`` | Files completed (backed up to repo) | +----------------------+------------------------------------------------------------+ |``total_bytes`` | Total number of bytes in backup set | +----------------------+------------------------------------------------------------+ -|``bytes_done`` | Number of bytes completed (backed up or confirmed in repo) | +|``bytes_done`` | Number of bytes completed (backed up to repo) | +----------------------+------------------------------------------------------------+ |``error_count`` | Number of errors | +----------------------+------------------------------------------------------------+ @@ -98,23 +124,23 @@ Error Verbose Status ^^^^^^^^^^^^^^ -Verbose status is a status line that supplements +Verbose status provides details about the progress, including details about backed up files. -+----------------------+-------------------------------------------+ -| ``message_type`` | Always "verbose_status" | -+----------------------+-------------------------------------------+ -| ``action`` | Either "new", "unchanged" or "modified" | -+----------------------+-------------------------------------------+ -| ``item`` | The item in question | -+----------------------+-------------------------------------------+ -| ``duration`` | How long it took, in seconds | -+----------------------+-------------------------------------------+ -| ``data_size`` | How big the item is | -+----------------------+-------------------------------------------+ -| ``metadata_size`` | How big the metadata is | -+----------------------+-------------------------------------------+ -| ``total_files`` | Total number of files | -+----------------------+-------------------------------------------+ ++----------------------+-----------------------------------------------------------+ +| ``message_type`` | Always "verbose_status" | ++----------------------+-----------------------------------------------------------+ +| ``action`` | Either "new", "unchanged", "modified" or "scan_finished" | ++----------------------+-----------------------------------------------------------+ +| ``item`` | The item in question | ++----------------------+-----------------------------------------------------------+ +| ``duration`` | How long it took, in seconds | ++----------------------+-----------------------------------------------------------+ +| ``data_size`` | How big the item is | ++----------------------+-----------------------------------------------------------+ +| ``metadata_size`` | How big the metadata is | ++----------------------+-----------------------------------------------------------+ +| ``total_files`` | Total number of files | ++----------------------+-----------------------------------------------------------+ Summary ^^^^^^^ @@ -148,69 +174,96 @@ Summary is the last output line in a successful backup. +---------------------------+---------------------------------------------------------+ | ``total_duration`` | Total time it took for the operation to complete | +---------------------------+---------------------------------------------------------+ -| ``snapshot_id`` | The short ID of the new snapshot | +| ``snapshot_id`` | ID of the new snapshot | +---------------------------+---------------------------------------------------------+ -snapshots ---------- - -The snapshots command returns a single JSON object, an array with the structure outlined below. - -+----------------+------------------------------------------------------------------------+ -| ``hostname`` | The hostname of the machine that's being backed up | -+----------------+------------------------------------------------------------------------+ -| ``username`` | The username that the backup command was run as | -+----------------+------------------------------------------------------------------------+ -| ``excludes`` | A list of paths and globs that were excluded from the backup | -+----------------+------------------------------------------------------------------------+ -| ``tags`` | A list of tags for the snapshot in question | -+----------------+------------------------------------------------------------------------+ -| ``id`` | The long snapshot ID | -+----------------+------------------------------------------------------------------------+ -| ``short_id`` | The short snapshot ID | -+----------------+------------------------------------------------------------------------+ -| ``time`` | The timestamp of when the backup was started | -+----------------+------------------------------------------------------------------------+ -| ``parent`` | The ID of the previous snapshot | -+----------------+------------------------------------------------------------------------+ -| ``tree`` | The ID of the root tree blob | -+----------------+------------------------------------------------------------------------+ -| ``paths`` | A list of paths that were included in the backup | -+----------------+------------------------------------------------------------------------+ cat --- -Cat will return data about various objects in the repository, already in json form. -By specifying ``--json``, it will suppress any non-json messages the command generates. +The ``cat`` command returns data about various objects in the repository, which +are stored in JSON form. Specifying ``--json`` or ``--quiet`` will suppress any +non-JSON messages the command generates. + + +diff +---- + +The ``diff`` command uses the JSON lines format with the following message types. + +change +^^^^^^ + ++------------------+--------------------------------------------------------------+ +| ``message_type`` | Always "change" | ++------------------+--------------------------------------------------------------+ +| ``path`` | Path that has changed | ++------------------+--------------------------------------------------------------+ +| ``modifier`` | Type of change, a concatenation of the following characters: | +| | "+" = added, "-" = removed, "T" = entry type changed, | +| | "M" = file content changed, "U" = metadata changed | ++------------------+--------------------------------------------------------------+ + +statistics +^^^^^^^^^^ + ++---------------------+----------------------------+ +| ``message_type`` | Always "statistics" | ++---------------------+----------------------------+ +| ``source_snapshot`` | ID of first snapshot | ++---------------------+----------------------------+ +| ``target_snapshot`` | ID of second snapshot | ++---------------------+----------------------------+ +| ``changed_files`` | Number of changed files | ++---------------------+----------------------------+ +| ``added`` | DiffStat object, see below | ++---------------------+----------------------------+ +| ``removed`` | DiffStat object, see below | ++---------------------+----------------------------+ + +DiffStat object + ++----------------+-------------------------------------------+ +| ``files`` | Number of changed files | ++----------------+-------------------------------------------+ +| ``dirs`` | Number of changed directories | ++----------------+-------------------------------------------+ +| ``others`` | Number of changed other directory entries | ++----------------+-------------------------------------------+ +| ``data_blobs`` | Number of data blobs | ++----------------+-------------------------------------------+ +| ``tree_blobs`` | Number of tree blobs | ++----------------+-------------------------------------------+ +| ``bytes`` | Number of bytes | ++----------------+-------------------------------------------+ + find ---- -The find command outputs an array of json objects with matches for your search term. These -matches are organized by snapshot. +The ``find`` command outputs a single JSON document containing an array of JSON +objects with matches for your search term. These matches are organized by snapshot. -Snapshot -^^^^^^^^ - -+-----------------+----------------------------------------------+ -| ``hits`` | The number of matches in the snapshot | -+-----------------+----------------------------------------------+ -| ``snapshot`` | The long ID of the snapshot | -+-----------------+----------------------------------------------+ -| ``matches`` | Array of JSON objects detailing a match. | -+-----------------+----------------------------------------------+ +If the ``--blob`` or ``--tree`` option is passed, then the output is an array of +Blob objects. -Match -^^^^^ ++-----------------+----------------------------------------------+ +| ``hits`` | Number of matches in the snapshot | ++-----------------+----------------------------------------------+ +| ``snapshot`` | ID of the snapshot | ++-----------------+----------------------------------------------+ +| ``matches`` | Array of Match objects detailing a match | ++-----------------+----------------------------------------------+ + +Match object +-----------------+----------------------------------------------+ | ``path`` | Object path | +-----------------+----------------------------------------------+ | ``permissions`` | UNIX permissions | +-----------------+----------------------------------------------+ -| ``type`` | what type it is e.g. file, dir, etc... | +| ``type`` | Object type e.g. file, dir, etc... | +-----------------+----------------------------------------------+ | ``atime`` | Access time | +-----------------+----------------------------------------------+ @@ -226,7 +279,7 @@ Match +-----------------+----------------------------------------------+ | ``mode`` | UNIX file mode, shorthand of ``permissions`` | +-----------------+----------------------------------------------+ -| ``device_id`` | Unique machine Identifier | +| ``device_id`` | OS specific device identifier | +-----------------+----------------------------------------------+ | ``links`` | Number of hardlinks | +-----------------+----------------------------------------------+ @@ -237,10 +290,106 @@ Match | ``size`` | Size of object in bytes | +-----------------+----------------------------------------------+ +Blob object + ++-----------------+--------------------------------------------+ +| ``object_type`` | Either "blob" or "tree" | ++-----------------+--------------------------------------------+ +| ``id`` | ID of found blob | ++-----------------+--------------------------------------------+ +| ``path`` | Path in snapshot | ++-----------------+--------------------------------------------+ +| ``parent_tree`` | Parent tree blob, only set for type "blob" | ++-----------------+--------------------------------------------+ +| ``snapshot`` | Snapshot ID | ++-----------------+--------------------------------------------+ +| ``time`` | Snapshot timestamp | ++-----------------+--------------------------------------------+ + + +forget +------ + +The ``forget`` command prints a single JSON document containing an array of +ForgetGroups. If specific snapshot IDs are specified, then no output is generated. + +The ``prune`` command does not yet support JSON such that ``forget --prune`` +results in a mix of JSON and text output. + +ForgetGroup +^^^^^^^^^^^ + ++-------------+-----------------------------------------------------------+ +| ``tags`` | Tags identifying the snapshot group | ++-------------+-----------------------------------------------------------+ +| ``host`` | Host identifying the snapshot group | ++-------------+-----------------------------------------------------------+ +| ``paths`` | Paths identifying the snapshot group | ++-------------+-----------------------------------------------------------+ +| ``keep`` | Array of Snapshot objects that are kept | ++-------------+-----------------------------------------------------------+ +| ``remove`` | Array of Snapshot objects that were removed | ++-------------+-----------------------------------------------------------+ +| ``reasons`` | Array of Reason objects describing why a snapshot is kept | ++-------------+-----------------------------------------------------------+ + +Snapshot object + ++----------------+--------------------------------------------------+ +| ``time`` | Timestamp of when the backup was started | ++----------------+--------------------------------------------------+ +| ``parent`` | ID of the parent snapshot | ++----------------+--------------------------------------------------+ +| ``tree`` | ID of the root tree blob | ++----------------+--------------------------------------------------+ +| ``paths`` | List of paths included in the backup | ++----------------+--------------------------------------------------+ +| ``hostname`` | Hostname of the backed up machine | ++----------------+--------------------------------------------------+ +| ``username`` | Username the backup command was run as | ++----------------+--------------------------------------------------+ +| ``uid`` | ID of owner | ++----------------+--------------------------------------------------+ +| ``gid`` | ID of group | ++----------------+--------------------------------------------------+ +| ``excludes`` | List of paths and globs excluded from the backup | ++----------------+--------------------------------------------------+ +| ``tags`` | List of tags for the snapshot in question | ++----------------+--------------------------------------------------+ +| ``id`` | Snapshot ID | ++----------------+--------------------------------------------------+ +| ``short_id`` | Snapshot ID, short form | ++----------------+--------------------------------------------------+ + +Reason object + ++----------------+---------------------------------------------------------+ +| ``snapshot`` | Snapshot object, without ``id`` and ``short_id`` fields | ++----------------+---------------------------------------------------------+ +| ``matches`` | Array containing descriptions of the matching criteria | ++----------------+---------------------------------------------------------+ +| ``counters`` | Object containing counters used by the policies | ++----------------+---------------------------------------------------------+ + + +init +---- + +The ``init`` command uses the JSON lines format, but only outputs a single message. + ++------------------+--------------------------------+ +| ``message_type`` | Always "initialized" | ++------------------+--------------------------------+ +| ``id`` | ID of the created repository | ++------------------+--------------------------------+ +| ``repository`` | URL of the repository | ++------------------+--------------------------------+ + + key list -------- -The key list command returns an array of objects with the following structure. +The ``key list`` command returns an array of objects with the following structure. +--------------+------------------------------------+ | ``current`` | Is currently used key? | @@ -254,41 +403,50 @@ The key list command returns an array of objects with the following structure. | ``created`` | Timestamp when it was created | +--------------+------------------------------------+ + ls -- -The ls command spits out a series of newline-separated JSON objects, -the nature of which can be determined by the ``struct_type`` field. +The ``ls`` command uses the JSON lines format with the following message types. +As an exception, the ``struct_type`` field is used to determine the message type. snapshot ^^^^^^^^ -+-----------------+-------------------------------------+ -| ``time`` | Snapshot time | -+-----------------+-------------------------------------+ -| ``tree`` | Snapshot tree root | -+-----------------+-------------------------------------+ -| ``paths`` | List of paths included in snapshot | -+-----------------+-------------------------------------+ -| ``hostname`` | Hostname of snapshot | -+-----------------+-------------------------------------+ -| ``username`` | User snapshot was run as | -+-----------------+-------------------------------------+ -| ``uid`` | ID of owner | -+-----------------+-------------------------------------+ -| ``gid`` | ID of group | -+-----------------+-------------------------------------+ -| ``id`` | Snapshot ID, long form | -+-----------------+-------------------------------------+ -| ``short_id`` | Snapshot ID, short form | -+-----------------+-------------------------------------+ -| ``struct_type`` | Always "snapshot" | -+-----------------+-------------------------------------+ ++----------------+--------------------------------------------------+ +| ``struct_type``| Always "snapshot" | ++----------------+--------------------------------------------------+ +| ``time`` | Timestamp of when the backup was started | ++----------------+--------------------------------------------------+ +| ``parent`` | ID of the parent snapshot | ++----------------+--------------------------------------------------+ +| ``tree`` | ID of the root tree blob | ++----------------+--------------------------------------------------+ +| ``paths`` | List of paths included in the backup | ++----------------+--------------------------------------------------+ +| ``hostname`` | Hostname of the backed up machine | ++----------------+--------------------------------------------------+ +| ``username`` | Username the backup command was run as | ++----------------+--------------------------------------------------+ +| ``uid`` | ID of owner | ++----------------+--------------------------------------------------+ +| ``gid`` | ID of group | ++----------------+--------------------------------------------------+ +| ``excludes`` | List of paths and globs excluded from the backup | ++----------------+--------------------------------------------------+ +| ``tags`` | List of tags for the snapshot in question | ++----------------+--------------------------------------------------+ +| ``id`` | Snapshot ID | ++----------------+--------------------------------------------------+ +| ``short_id`` | Snapshot ID, short form | ++----------------+--------------------------------------------------+ node ^^^^ ++-----------------+--------------------------+ +| ``struct_type`` | Always "node" | +-----------------+--------------------------+ | ``name`` | Node name | +-----------------+--------------------------+ @@ -310,16 +468,60 @@ node +-----------------+--------------------------+ | ``ctime`` | Node creation time | +-----------------+--------------------------+ -| ``struct_type`` | Always "node" | -+-----------------+--------------------------+ + + +snapshots +--------- + +The snapshots command returns a single JSON object, an array with objects of the structure outlined below. + ++----------------+--------------------------------------------------+ +| ``time`` | Timestamp of when the backup was started | ++----------------+--------------------------------------------------+ +| ``parent`` | ID of the parent snapshot | ++----------------+--------------------------------------------------+ +| ``tree`` | ID of the root tree blob | ++----------------+--------------------------------------------------+ +| ``paths`` | List of paths included in the backup | ++----------------+--------------------------------------------------+ +| ``hostname`` | Hostname of the backed up machine | ++----------------+--------------------------------------------------+ +| ``username`` | Username the backup command was run as | ++----------------+--------------------------------------------------+ +| ``uid`` | ID of owner | ++----------------+--------------------------------------------------+ +| ``gid`` | ID of group | ++----------------+--------------------------------------------------+ +| ``excludes`` | List of paths and globs excluded from the backup | ++----------------+--------------------------------------------------+ +| ``tags`` | List of tags for the snapshot in question | ++----------------+--------------------------------------------------+ +| ``id`` | Snapshot ID | ++----------------+--------------------------------------------------+ +| ``short_id`` | Snapshot ID, short form | ++----------------+--------------------------------------------------+ + stats ----- -+----------------------+---------------------------------------------+ -| ``total_size`` | Repository size in bytes | -+----------------------+---------------------------------------------+ -| ``total_file_count`` | Number of files backed up in the repository | -+----------------------+---------------------------------------------+ -| ``total_blob_count`` | Number of blobs in the repository | -+----------------------+---------------------------------------------+ +The snapshots command returns a single JSON object. + ++------------------------------+-----------------------------------------------------+ +| ``total_size`` | Repository size in bytes | ++------------------------------+-----------------------------------------------------+ +| ``total_file_count`` | Number of files backed up in the repository | ++------------------------------+-----------------------------------------------------+ +| ``total_blob_count`` | Number of blobs in the repository | ++------------------------------+-----------------------------------------------------+ +| ``snapshots_count`` | Number of processed snapshots | ++------------------------------+-----------------------------------------------------+ +| ``total_uncompressed_size`` | Repository size in bytes if blobs were uncompressed | ++------------------------------+-----------------------------------------------------+ +| ``compression_ratio`` | Factor by which the already compressed data | +| | has shrunk due to compression | ++------------------------------+-----------------------------------------------------+ +| ``compression_progress`` | Percentage of already compressed data | ++------------------------------+-----------------------------------------------------+ +| ``compression_space_saving`` | Overall space saving due to compression | ++------------------------------+-----------------------------------------------------+ From b34ce57dd4d48059d183d92afc6d14c0e7a276b6 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 17 Jun 2023 21:47:17 +0200 Subject: [PATCH 83/90] doc: describe JSON output of restore command --- doc/075_scripting.rst | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index 9855b9ffb..85c67ab38 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -470,6 +470,49 @@ node +-----------------+--------------------------+ +restore +------- + +The ``restore`` command uses the JSON lines format with the following message types. + +Status +^^^^^^ + ++----------------------+------------------------------------------------------------+ +|``message_type`` | Always "status" | ++----------------------+------------------------------------------------------------+ +|``seconds_elapsed`` | Time since restore started | ++----------------------+------------------------------------------------------------+ +|``percent_done`` | Percentage of data backed up (bytes_restored/total_bytes) | ++----------------------+------------------------------------------------------------+ +|``total_files`` | Total number of files detected | ++----------------------+------------------------------------------------------------+ +|``files_restored`` | Files restored | ++----------------------+------------------------------------------------------------+ +|``total_bytes`` | Total number of bytes in restore set | ++----------------------+------------------------------------------------------------+ +|``bytes_restored`` | Number of bytes restored | ++----------------------+------------------------------------------------------------+ + + +Summary +^^^^^^^ + ++----------------------+------------------------------------------------------------+ +|``message_type`` | Always "summary" | ++----------------------+------------------------------------------------------------+ +|``seconds_elapsed`` | Time since restore started | ++----------------------+------------------------------------------------------------+ +|``total_files`` | Total number of files detected | ++----------------------+------------------------------------------------------------+ +|``files_restored`` | Files restored | ++----------------------+------------------------------------------------------------+ +|``total_bytes`` | Total number of bytes in restore set | ++----------------------+------------------------------------------------------------+ +|``bytes_restored`` | Number of bytes restored | ++----------------------+------------------------------------------------------------+ + + snapshots --------- From 229c7b24a4385d0a4397d5b9d791b2cdaf021b45 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 8 Jul 2023 23:29:46 +0200 Subject: [PATCH 84/90] doc: add program_version field of snapshot JSON output --- doc/075_scripting.rst | 104 ++++++++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 50 deletions(-) diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index 85c67ab38..a4b983d7c 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -335,31 +335,33 @@ ForgetGroup Snapshot object -+----------------+--------------------------------------------------+ -| ``time`` | Timestamp of when the backup was started | -+----------------+--------------------------------------------------+ -| ``parent`` | ID of the parent snapshot | -+----------------+--------------------------------------------------+ -| ``tree`` | ID of the root tree blob | -+----------------+--------------------------------------------------+ -| ``paths`` | List of paths included in the backup | -+----------------+--------------------------------------------------+ -| ``hostname`` | Hostname of the backed up machine | -+----------------+--------------------------------------------------+ -| ``username`` | Username the backup command was run as | -+----------------+--------------------------------------------------+ -| ``uid`` | ID of owner | -+----------------+--------------------------------------------------+ -| ``gid`` | ID of group | -+----------------+--------------------------------------------------+ -| ``excludes`` | List of paths and globs excluded from the backup | -+----------------+--------------------------------------------------+ -| ``tags`` | List of tags for the snapshot in question | -+----------------+--------------------------------------------------+ -| ``id`` | Snapshot ID | -+----------------+--------------------------------------------------+ -| ``short_id`` | Snapshot ID, short form | -+----------------+--------------------------------------------------+ ++---------------------+--------------------------------------------------+ +| ``time`` | Timestamp of when the backup was started | ++---------------------+--------------------------------------------------+ +| ``parent`` | ID of the parent snapshot | ++---------------------+--------------------------------------------------+ +| ``tree`` | ID of the root tree blob | ++---------------------+--------------------------------------------------+ +| ``paths`` | List of paths included in the backup | ++---------------------+--------------------------------------------------+ +| ``hostname`` | Hostname of the backed up machine | ++---------------------+--------------------------------------------------+ +| ``username`` | Username the backup command was run as | ++---------------------+--------------------------------------------------+ +| ``uid`` | ID of owner | ++---------------------+--------------------------------------------------+ +| ``gid`` | ID of group | ++---------------------+--------------------------------------------------+ +| ``excludes`` | List of paths and globs excluded from the backup | ++---------------------+--------------------------------------------------+ +| ``tags`` | List of tags for the snapshot in question | ++---------------------+--------------------------------------------------+ +| ``program_version`` | restic version used to create snapshot | ++---------------------+--------------------------------------------------+ +| ``id`` | Snapshot ID | ++---------------------+--------------------------------------------------+ +| ``short_id`` | Snapshot ID, short form | ++---------------------+--------------------------------------------------+ Reason object @@ -518,31 +520,33 @@ snapshots The snapshots command returns a single JSON object, an array with objects of the structure outlined below. -+----------------+--------------------------------------------------+ -| ``time`` | Timestamp of when the backup was started | -+----------------+--------------------------------------------------+ -| ``parent`` | ID of the parent snapshot | -+----------------+--------------------------------------------------+ -| ``tree`` | ID of the root tree blob | -+----------------+--------------------------------------------------+ -| ``paths`` | List of paths included in the backup | -+----------------+--------------------------------------------------+ -| ``hostname`` | Hostname of the backed up machine | -+----------------+--------------------------------------------------+ -| ``username`` | Username the backup command was run as | -+----------------+--------------------------------------------------+ -| ``uid`` | ID of owner | -+----------------+--------------------------------------------------+ -| ``gid`` | ID of group | -+----------------+--------------------------------------------------+ -| ``excludes`` | List of paths and globs excluded from the backup | -+----------------+--------------------------------------------------+ -| ``tags`` | List of tags for the snapshot in question | -+----------------+--------------------------------------------------+ -| ``id`` | Snapshot ID | -+----------------+--------------------------------------------------+ -| ``short_id`` | Snapshot ID, short form | -+----------------+--------------------------------------------------+ ++---------------------+--------------------------------------------------+ +| ``time`` | Timestamp of when the backup was started | ++---------------------+--------------------------------------------------+ +| ``parent`` | ID of the parent snapshot | ++---------------------+--------------------------------------------------+ +| ``tree`` | ID of the root tree blob | ++---------------------+--------------------------------------------------+ +| ``paths`` | List of paths included in the backup | ++---------------------+--------------------------------------------------+ +| ``hostname`` | Hostname of the backed up machine | ++---------------------+--------------------------------------------------+ +| ``username`` | Username the backup command was run as | ++---------------------+--------------------------------------------------+ +| ``uid`` | ID of owner | ++---------------------+--------------------------------------------------+ +| ``gid`` | ID of group | ++---------------------+--------------------------------------------------+ +| ``excludes`` | List of paths and globs excluded from the backup | ++---------------------+--------------------------------------------------+ +| ``tags`` | List of tags for the snapshot in question | ++---------------------+--------------------------------------------------+ +| ``program_version`` | restic version used to create snapshot | ++---------------------+--------------------------------------------------+ +| ``id`` | Snapshot ID | ++---------------------+--------------------------------------------------+ +| ``short_id`` | Snapshot ID, short form | ++---------------------+--------------------------------------------------+ stats From 8a120c8800156356a515a97a6a13738561da1da3 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 8 Jul 2023 23:51:39 +0200 Subject: [PATCH 85/90] CI: Enable missing CI tests for Github merge queue --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8e8a5b099..0ec8c072c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,6 +7,7 @@ on: # run tests for all pull requests pull_request: + merge_group: permissions: contents: read From 1ce599d2ae68285426e251031cb98a87cb530ff6 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 9 Jul 2023 14:15:23 +0200 Subject: [PATCH 86/90] Fix handling of empty cacert environment variable This resulted in a "empty filename for root certificate supplied" error. --- cmd/restic/global.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 487fa9673..07412925e 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -151,7 +151,9 @@ func init() { globalOptions.PasswordFile = os.Getenv("RESTIC_PASSWORD_FILE") globalOptions.KeyHint = os.Getenv("RESTIC_KEY_HINT") globalOptions.PasswordCommand = os.Getenv("RESTIC_PASSWORD_COMMAND") - globalOptions.RootCertFilenames = strings.Split(os.Getenv("RESTIC_CACERT"), ",") + if os.Getenv("RESTIC_CACERT") != "" { + globalOptions.RootCertFilenames = strings.Split(os.Getenv("RESTIC_CACERT"), ",") + } globalOptions.TLSClientCertKeyFilename = os.Getenv("RESTIC_TLS_CLIENT_CERT") comp := os.Getenv("RESTIC_COMPRESSION") if comp != "" { From 89fbd39e59f4febb75575af227c8c12eee5c72b9 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 9 Jul 2023 14:17:43 +0200 Subject: [PATCH 87/90] Don't print stacktrace on invalid cacert option --- cmd/restic/global.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 07412925e..31d5aac16 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -584,7 +584,7 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio rt, err := backend.Transport(globalOptions.TransportOptions) if err != nil { - return nil, err + return nil, errors.Fatal(err.Error()) } // wrap the transport so that the throughput via HTTP is limited @@ -640,7 +640,7 @@ func create(ctx context.Context, s string, gopts GlobalOptions, opts options.Opt rt, err := backend.Transport(globalOptions.TransportOptions) if err != nil { - return nil, err + return nil, errors.Fatal(err.Error()) } factory := gopts.backends.Lookup(loc.Scheme) From c158741e2ea08997f37b71db025bd5f595298475 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 9 Jul 2023 14:29:43 +0200 Subject: [PATCH 88/90] CI: add minimal CLI test Just create a repository and run a minimal backup. --- .github/workflows/tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0ec8c072c..bb9945891 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -141,6 +141,14 @@ jobs: run: | go run build.go + - name: Minimal test + run: | + ./restic init + ./restic backup . + env: + RESTIC_REPOSITORY: ../testrepo + RESTIC_PASSWORD: password + - name: Run local Tests env: RESTIC_TEST_FUSE: ${{ matrix.test_fuse }} From e990d3d483a7a257adf615979f357fe5cdc90f78 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 8 Jun 2023 21:54:49 +0200 Subject: [PATCH 89/90] azure: Support authentication using managed / workload identity See https://github.com/Azure/azure-sdk-for-go/tree/sdk/azidentity/v1.3.0/sdk/azidentity --- changelog/unreleased/issue-3698 | 8 ++++++++ go.mod | 6 +++++- go.sum | 17 ++++++++++------- internal/backend/azure/azure.go | 12 +++++++++++- 4 files changed, 34 insertions(+), 9 deletions(-) create mode 100644 changelog/unreleased/issue-3698 diff --git a/changelog/unreleased/issue-3698 b/changelog/unreleased/issue-3698 new file mode 100644 index 000000000..0851d3756 --- /dev/null +++ b/changelog/unreleased/issue-3698 @@ -0,0 +1,8 @@ +Enhancement: Add support for Managed / Worload Identity to azure backend + +Restic now additionally supports authenticating to Azure using Workload +Identity or Managed Identity credentials which are automatically injected in +several environments such as a managed Kubernetes cluster. + +https://github.com/restic/restic/issues/3698 +https://github.com/restic/restic/pull/4029 diff --git a/go.mod b/go.mod index 1554eca40..867e7dc91 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/restic/restic require ( cloud.google.com/go/storage v1.30.1 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 github.com/anacrolix/fuse v0.2.0 github.com/cenkalti/backoff/v4 v4.2.0 @@ -40,10 +41,11 @@ require ( cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v0.13.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/dnaeon/go-vcr v1.2.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/fgprof v0.9.3 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b // indirect @@ -55,9 +57,11 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/kr/fs v0.1.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/rs/xid v1.5.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.9.2 // indirect diff --git a/go.sum b/go.sum index 7893b5e39..6ca1f93e6 100644 --- a/go.sum +++ b/go.sum @@ -12,12 +12,14 @@ cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/o cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 h1:SEy2xmstIphdPwNBUi7uhvjyjhVKISfwjfOJmuy7kg4= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= -github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/anacrolix/fuse v0.2.0 h1:pc+To78kI2d/WUjIyrsdqeJQAesuwpGxlI3h1nAv3Do= @@ -45,7 +47,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= @@ -63,7 +64,8 @@ github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNu github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= @@ -128,6 +130,7 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5 h1:OUlGa6AAolmjyPtILbMJ8vHayz5wd4wBUloheGcMhfA= github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.56 h1:pkZplIEHu8vinjkmhsexcXpWth2tjVLphrTZx6fBVZY= @@ -139,10 +142,10 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/ncw/swift/v2 v2.0.1 h1:q1IN8hNViXEv8Zvg3Xdis4a3c4IlIGezkYz09zQL5J0= github.com/ncw/swift/v2 v2.0.1/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg= -github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= @@ -238,6 +241,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -318,7 +322,6 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index 661dd505d..50be63d5a 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -21,6 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" @@ -101,7 +102,16 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { return nil, errors.Wrap(err, "NewAccountSASClientFromEndpointToken") } } else { - return nil, errors.New("no azure authentication information found") + debug.Log(" - using DefaultAzureCredential") + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + return nil, errors.Wrap(err, "NewDefaultAzureCredential") + } + + client, err = azContainer.NewClient(url, cred, opts) + if err != nil { + return nil, errors.Wrap(err, "NewClient") + } } be := &Backend{ From c9f506925c12b59f2e520daa82d246eca67aed0c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 9 Jul 2023 20:56:21 +0200 Subject: [PATCH 90/90] azure: Document additional auth options --- doc/030_preparing_a_new_repo.rst | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index c944264c8..04c189d07 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -523,20 +523,24 @@ Microsoft Azure Blob Storage **************************** You can also store backups on Microsoft Azure Blob Storage. Export the Azure -Blob Storage account name and key as follows: +Blob Storage account name: .. code-block:: console $ export AZURE_ACCOUNT_NAME= + +For authentication export one of the following variables: + +.. code-block:: console + + # For storage account key $ export AZURE_ACCOUNT_KEY= - -or - -.. code-block:: console - - $ export AZURE_ACCOUNT_NAME= + # For SAS $ export AZURE_ACCOUNT_SAS= +Alternatively, if run on Azure, restic will automatically uses service accounts configured +via the standard environment variables or Workload / Managed Identities. + Restic will by default use Azure's global domain ``core.windows.net`` as endpoint suffix. You can specify other suffixes as follows: