mirror of
https://github.com/restic/restic.git
synced 2025-03-30 00:00:14 +01:00
backend/test: parallelize slow tests
This commit is contained in:
Michael Eischer
parent
c36970074d
commit
4640b3c41a
1 changed files with 125 additions and 109 deletions
|
|
@ -10,11 +10,13 @@ import (
|
|||
"os"
|
||||
"reflect"
|
||||
"sort"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"github.com/restic/restic/internal/test"
|
||||
|
||||
|
|
@ -276,16 +278,26 @@ func (s *Suite[C]) TestList(t *testing.T) {
|
|||
}
|
||||
|
||||
list1 := make(map[restic.ID]int64)
|
||||
var m sync.Mutex
|
||||
|
||||
wg, ctx := errgroup.WithContext(context.TODO())
|
||||
for i := 0; i < numTestFiles; i++ {
|
||||
data := test.Random(random.Int(), random.Intn(100)+55)
|
||||
id := restic.Hash(data)
|
||||
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
|
||||
err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list1[id] = int64(len(data))
|
||||
wg.Go(func() error {
|
||||
id := restic.Hash(data)
|
||||
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
|
||||
err := b.Save(ctx, h, backend.NewByteReader(data, b.Hasher()))
|
||||
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
list1[id] = int64(len(data))
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
err = wg.Wait()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("wrote %v files", len(list1))
|
||||
|
|
@ -777,125 +789,129 @@ func delayedList(t testing.TB, b backend.Backend, tpe backend.FileType, max int,
|
|||
|
||||
// TestBackend tests all functions of the backend.
|
||||
func (s *Suite[C]) TestBackend(t *testing.T) {
|
||||
b := s.open(t)
|
||||
defer s.close(t, b)
|
||||
|
||||
test.Assert(t, !b.IsNotExist(nil), "IsNotExist() recognized nil error")
|
||||
test.Assert(t, !b.IsPermanentError(nil), "IsPermanentError() recognized nil error")
|
||||
|
||||
for _, tpe := range []backend.FileType{
|
||||
backend.PackFile, backend.KeyFile, backend.LockFile,
|
||||
backend.SnapshotFile, backend.IndexFile,
|
||||
} {
|
||||
// detect non-existing files
|
||||
for _, ts := range testStrings {
|
||||
id, err := restic.ParseID(ts.id)
|
||||
test.OK(t, err)
|
||||
t.Run(tpe.String(), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// test if blob is already in repository
|
||||
h := backend.Handle{Type: tpe, Name: id.String()}
|
||||
ret, err := beTest(context.TODO(), b, h)
|
||||
test.OK(t, err)
|
||||
test.Assert(t, !ret, "blob was found to exist before creating")
|
||||
b := s.open(t)
|
||||
defer s.close(t, b)
|
||||
|
||||
// try to stat a not existing blob
|
||||
_, err = b.Stat(context.TODO(), h)
|
||||
test.Assert(t, err != nil, "blob data could be extracted before creation")
|
||||
test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize Stat() error: %v", err)
|
||||
test.Assert(t, b.IsPermanentError(err), "IsPermanentError() did not recognize Stat() error: %v", err)
|
||||
test.Assert(t, !b.IsNotExist(nil), "IsNotExist() recognized nil error")
|
||||
test.Assert(t, !b.IsPermanentError(nil), "IsPermanentError() recognized nil error")
|
||||
|
||||
// try to read not existing blob
|
||||
err = testLoad(b, h)
|
||||
test.Assert(t, err != nil, "blob could be read before creation")
|
||||
test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize Load() error: %v", err)
|
||||
test.Assert(t, b.IsPermanentError(err), "IsPermanentError() did not recognize Load() error: %v", err)
|
||||
// detect non-existing files
|
||||
for _, ts := range testStrings {
|
||||
id, err := restic.ParseID(ts.id)
|
||||
test.OK(t, err)
|
||||
|
||||
// try to get string out, should fail
|
||||
ret, err = beTest(context.TODO(), b, h)
|
||||
test.OK(t, err)
|
||||
test.Assert(t, !ret, "id %q was found (but should not have)", ts.id)
|
||||
}
|
||||
// test if blob is already in repository
|
||||
h := backend.Handle{Type: tpe, Name: id.String()}
|
||||
ret, err := beTest(context.TODO(), b, h)
|
||||
test.OK(t, err)
|
||||
test.Assert(t, !ret, "blob was found to exist before creating")
|
||||
|
||||
// add files
|
||||
for _, ts := range testStrings {
|
||||
store(t, b, tpe, []byte(ts.data))
|
||||
// try to stat a not existing blob
|
||||
_, err = b.Stat(context.TODO(), h)
|
||||
test.Assert(t, err != nil, "blob data could be extracted before creation")
|
||||
test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize Stat() error: %v", err)
|
||||
test.Assert(t, b.IsPermanentError(err), "IsPermanentError() did not recognize Stat() error: %v", err)
|
||||
|
||||
// test Load()
|
||||
// try to read not existing blob
|
||||
err = testLoad(b, h)
|
||||
test.Assert(t, err != nil, "blob could be read before creation")
|
||||
test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize Load() error: %v", err)
|
||||
test.Assert(t, b.IsPermanentError(err), "IsPermanentError() did not recognize Load() error: %v", err)
|
||||
|
||||
// try to get string out, should fail
|
||||
ret, err = beTest(context.TODO(), b, h)
|
||||
test.OK(t, err)
|
||||
test.Assert(t, !ret, "id %q was found (but should not have)", ts.id)
|
||||
}
|
||||
|
||||
// add files
|
||||
for _, ts := range testStrings {
|
||||
store(t, b, tpe, []byte(ts.data))
|
||||
|
||||
// test Load()
|
||||
h := backend.Handle{Type: tpe, Name: ts.id}
|
||||
buf, err := LoadAll(context.TODO(), b, h)
|
||||
test.OK(t, err)
|
||||
test.Equals(t, ts.data, string(buf))
|
||||
|
||||
// try to read it out with an offset and a length
|
||||
start := 1
|
||||
end := len(ts.data) - 2
|
||||
length := end - start
|
||||
|
||||
buf2 := make([]byte, length)
|
||||
var n int
|
||||
err = b.Load(context.TODO(), h, len(buf2), int64(start), func(rd io.Reader) (ierr error) {
|
||||
n, ierr = io.ReadFull(rd, buf2)
|
||||
return ierr
|
||||
})
|
||||
test.OK(t, err)
|
||||
test.OK(t, err)
|
||||
test.Equals(t, len(buf2), n)
|
||||
test.Equals(t, ts.data[start:end], string(buf2))
|
||||
}
|
||||
|
||||
// test adding the first file again
|
||||
ts := testStrings[0]
|
||||
h := backend.Handle{Type: tpe, Name: ts.id}
|
||||
buf, err := LoadAll(context.TODO(), b, h)
|
||||
test.OK(t, err)
|
||||
test.Equals(t, ts.data, string(buf))
|
||||
|
||||
// try to read it out with an offset and a length
|
||||
start := 1
|
||||
end := len(ts.data) - 2
|
||||
length := end - start
|
||||
|
||||
buf2 := make([]byte, length)
|
||||
var n int
|
||||
err = b.Load(context.TODO(), h, len(buf2), int64(start), func(rd io.Reader) (ierr error) {
|
||||
n, ierr = io.ReadFull(rd, buf2)
|
||||
return ierr
|
||||
})
|
||||
test.OK(t, err)
|
||||
test.OK(t, err)
|
||||
test.Equals(t, len(buf2), n)
|
||||
test.Equals(t, ts.data[start:end], string(buf2))
|
||||
}
|
||||
|
||||
// test adding the first file again
|
||||
ts := testStrings[0]
|
||||
h := backend.Handle{Type: tpe, Name: ts.id}
|
||||
|
||||
// remove and recreate
|
||||
err := s.delayedRemove(t, b, h)
|
||||
test.OK(t, err)
|
||||
|
||||
// test that the blob is gone
|
||||
ok, err := beTest(context.TODO(), b, h)
|
||||
test.OK(t, err)
|
||||
test.Assert(t, !ok, "removed blob still present")
|
||||
|
||||
// create blob
|
||||
err = b.Save(context.TODO(), h, backend.NewByteReader([]byte(ts.data), b.Hasher()))
|
||||
test.OK(t, err)
|
||||
|
||||
// list items
|
||||
IDs := restic.IDs{}
|
||||
|
||||
for _, ts := range testStrings {
|
||||
id, err := restic.ParseID(ts.id)
|
||||
test.OK(t, err)
|
||||
IDs = append(IDs, id)
|
||||
}
|
||||
|
||||
list := delayedList(t, b, tpe, len(IDs), s.WaitForDelayedRemoval)
|
||||
if len(IDs) != len(list) {
|
||||
t.Fatalf("wrong number of IDs returned: want %d, got %d", len(IDs), len(list))
|
||||
}
|
||||
|
||||
sort.Sort(IDs)
|
||||
sort.Sort(list)
|
||||
|
||||
if !reflect.DeepEqual(IDs, list) {
|
||||
t.Fatalf("lists aren't equal, want:\n %v\n got:\n%v\n", IDs, list)
|
||||
}
|
||||
|
||||
var handles []backend.Handle
|
||||
for _, ts := range testStrings {
|
||||
id, err := restic.ParseID(ts.id)
|
||||
// remove and recreate
|
||||
err := s.delayedRemove(t, b, h)
|
||||
test.OK(t, err)
|
||||
|
||||
h := backend.Handle{Type: tpe, Name: id.String()}
|
||||
|
||||
found, err := beTest(context.TODO(), b, h)
|
||||
// test that the blob is gone
|
||||
ok, err := beTest(context.TODO(), b, h)
|
||||
test.OK(t, err)
|
||||
test.Assert(t, found, fmt.Sprintf("id %v/%q not found", tpe, id))
|
||||
test.Assert(t, !ok, "removed blob still present")
|
||||
|
||||
handles = append(handles, h)
|
||||
}
|
||||
// create blob
|
||||
err = b.Save(context.TODO(), h, backend.NewByteReader([]byte(ts.data), b.Hasher()))
|
||||
test.OK(t, err)
|
||||
|
||||
test.OK(t, s.delayedRemove(t, b, handles...))
|
||||
// list items
|
||||
IDs := restic.IDs{}
|
||||
|
||||
for _, ts := range testStrings {
|
||||
id, err := restic.ParseID(ts.id)
|
||||
test.OK(t, err)
|
||||
IDs = append(IDs, id)
|
||||
}
|
||||
|
||||
list := delayedList(t, b, tpe, len(IDs), s.WaitForDelayedRemoval)
|
||||
if len(IDs) != len(list) {
|
||||
t.Fatalf("wrong number of IDs returned: want %d, got %d", len(IDs), len(list))
|
||||
}
|
||||
|
||||
sort.Sort(IDs)
|
||||
sort.Sort(list)
|
||||
|
||||
if !reflect.DeepEqual(IDs, list) {
|
||||
t.Fatalf("lists aren't equal, want:\n %v\n got:\n%v\n", IDs, list)
|
||||
}
|
||||
|
||||
var handles []backend.Handle
|
||||
for _, ts := range testStrings {
|
||||
id, err := restic.ParseID(ts.id)
|
||||
test.OK(t, err)
|
||||
|
||||
h := backend.Handle{Type: tpe, Name: id.String()}
|
||||
|
||||
found, err := beTest(context.TODO(), b, h)
|
||||
test.OK(t, err)
|
||||
test.Assert(t, found, fmt.Sprintf("id %v/%q not found", tpe, id))
|
||||
|
||||
handles = append(handles, h)
|
||||
}
|
||||
|
||||
test.OK(t, s.delayedRemove(t, b, handles...))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue