backend/test: parallelize slow tests

internal/backend/test/tests.go

@@ -10,11 +10,13 @@ import (
 	"os"
 	"reflect"
 	"sort"
+	"sync"
 	"testing"
 	"time"
 
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
+	"golang.org/x/sync/errgroup"
 
 	"github.com/restic/restic/internal/test"
 
@@ -276,16 +278,26 @@ func (s *Suite[C]) TestList(t *testing.T) {
 	}
 
 	list1 := make(map[restic.ID]int64)
+	var m sync.Mutex
 
+	wg, ctx := errgroup.WithContext(context.TODO())
 	for i := 0; i < numTestFiles; i++ {
 		data := test.Random(random.Int(), random.Intn(100)+55)
-		id := restic.Hash(data)
+		wg.Go(func() error {
-		h := backend.Handle{Type: backend.PackFile, Name: id.String()}
+			id := restic.Hash(data)
-		err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
+			h := backend.Handle{Type: backend.PackFile, Name: id.String()}
-		if err != nil {
+			err := b.Save(ctx, h, backend.NewByteReader(data, b.Hasher()))
-			t.Fatal(err)
+
-		}
+			m.Lock()
-		list1[id] = int64(len(data))
+			defer m.Unlock()
+			list1[id] = int64(len(data))
+			return err
+		})
+	}
+
+	err = wg.Wait()
+	if err != nil {
+		t.Fatal(err)
 	}
 
 	t.Logf("wrote %v files", len(list1))
@@ -777,125 +789,129 @@ func delayedList(t testing.TB, b backend.Backend, tpe backend.FileType, max int,
 
 // TestBackend tests all functions of the backend.
 func (s *Suite[C]) TestBackend(t *testing.T) {
-	b := s.open(t)
-	defer s.close(t, b)
-
-	test.Assert(t, !b.IsNotExist(nil), "IsNotExist() recognized nil error")
-	test.Assert(t, !b.IsPermanentError(nil), "IsPermanentError() recognized nil error")
-
 	for _, tpe := range []backend.FileType{
 		backend.PackFile, backend.KeyFile, backend.LockFile,
 		backend.SnapshotFile, backend.IndexFile,
 	} {
-		// detect non-existing files
+		t.Run(tpe.String(), func(t *testing.T) {
-		for _, ts := range testStrings {
+			t.Parallel()
-			id, err := restic.ParseID(ts.id)
-			test.OK(t, err)
 
-			// test if blob is already in repository
+			b := s.open(t)
-			h := backend.Handle{Type: tpe, Name: id.String()}
+			defer s.close(t, b)
-			ret, err := beTest(context.TODO(), b, h)
-			test.OK(t, err)
-			test.Assert(t, !ret, "blob was found to exist before creating")
 
-			// try to stat a not existing blob
+			test.Assert(t, !b.IsNotExist(nil), "IsNotExist() recognized nil error")
-			_, err = b.Stat(context.TODO(), h)
+			test.Assert(t, !b.IsPermanentError(nil), "IsPermanentError() recognized nil error")
-			test.Assert(t, err != nil, "blob data could be extracted before creation")
-			test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize Stat() error: %v", err)
-			test.Assert(t, b.IsPermanentError(err), "IsPermanentError() did not recognize Stat() error: %v", err)
 
-			// try to read not existing blob
+			// detect non-existing files
-			err = testLoad(b, h)
+			for _, ts := range testStrings {
-			test.Assert(t, err != nil, "blob could be read before creation")
+				id, err := restic.ParseID(ts.id)
-			test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize Load() error: %v", err)
+				test.OK(t, err)
-			test.Assert(t, b.IsPermanentError(err), "IsPermanentError() did not recognize Load() error: %v", err)
 
-			// try to get string out, should fail
+				// test if blob is already in repository
-			ret, err = beTest(context.TODO(), b, h)
+				h := backend.Handle{Type: tpe, Name: id.String()}
-			test.OK(t, err)
+				ret, err := beTest(context.TODO(), b, h)
-			test.Assert(t, !ret, "id %q was found (but should not have)", ts.id)
+				test.OK(t, err)
-		}
+				test.Assert(t, !ret, "blob was found to exist before creating")
 
-		// add files
+				// try to stat a not existing blob
-		for _, ts := range testStrings {
+				_, err = b.Stat(context.TODO(), h)
-			store(t, b, tpe, []byte(ts.data))
+				test.Assert(t, err != nil, "blob data could be extracted before creation")
+				test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize Stat() error: %v", err)
+				test.Assert(t, b.IsPermanentError(err), "IsPermanentError() did not recognize Stat() error: %v", err)
 
-			// test Load()
+				// try to read not existing blob
+				err = testLoad(b, h)
+				test.Assert(t, err != nil, "blob could be read before creation")
+				test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize Load() error: %v", err)
+				test.Assert(t, b.IsPermanentError(err), "IsPermanentError() did not recognize Load() error: %v", err)
+
+				// try to get string out, should fail
+				ret, err = beTest(context.TODO(), b, h)
+				test.OK(t, err)
+				test.Assert(t, !ret, "id %q was found (but should not have)", ts.id)
+			}
+
+			// add files
+			for _, ts := range testStrings {
+				store(t, b, tpe, []byte(ts.data))
+
+				// test Load()
+				h := backend.Handle{Type: tpe, Name: ts.id}
+				buf, err := LoadAll(context.TODO(), b, h)
+				test.OK(t, err)
+				test.Equals(t, ts.data, string(buf))
+
+				// try to read it out with an offset and a length
+				start := 1
+				end := len(ts.data) - 2
+				length := end - start
+
+				buf2 := make([]byte, length)
+				var n int
+				err = b.Load(context.TODO(), h, len(buf2), int64(start), func(rd io.Reader) (ierr error) {
+					n, ierr = io.ReadFull(rd, buf2)
+					return ierr
+				})
+				test.OK(t, err)
+				test.OK(t, err)
+				test.Equals(t, len(buf2), n)
+				test.Equals(t, ts.data[start:end], string(buf2))
+			}
+
+			// test adding the first file again
+			ts := testStrings[0]
 			h := backend.Handle{Type: tpe, Name: ts.id}
-			buf, err := LoadAll(context.TODO(), b, h)
-			test.OK(t, err)
-			test.Equals(t, ts.data, string(buf))
 
-			// try to read it out with an offset and a length
+			// remove and recreate
-			start := 1
+			err := s.delayedRemove(t, b, h)
-			end := len(ts.data) - 2
-			length := end - start
-
-			buf2 := make([]byte, length)
-			var n int
-			err = b.Load(context.TODO(), h, len(buf2), int64(start), func(rd io.Reader) (ierr error) {
-				n, ierr = io.ReadFull(rd, buf2)
-				return ierr
-			})
-			test.OK(t, err)
-			test.OK(t, err)
-			test.Equals(t, len(buf2), n)
-			test.Equals(t, ts.data[start:end], string(buf2))
-		}
-
-		// test adding the first file again
-		ts := testStrings[0]
-		h := backend.Handle{Type: tpe, Name: ts.id}
-
-		// remove and recreate
-		err := s.delayedRemove(t, b, h)
-		test.OK(t, err)
-
-		// test that the blob is gone
-		ok, err := beTest(context.TODO(), b, h)
-		test.OK(t, err)
-		test.Assert(t, !ok, "removed blob still present")
-
-		// create blob
-		err = b.Save(context.TODO(), h, backend.NewByteReader([]byte(ts.data), b.Hasher()))
-		test.OK(t, err)
-
-		// list items
-		IDs := restic.IDs{}
-
-		for _, ts := range testStrings {
-			id, err := restic.ParseID(ts.id)
-			test.OK(t, err)
-			IDs = append(IDs, id)
-		}
-
-		list := delayedList(t, b, tpe, len(IDs), s.WaitForDelayedRemoval)
-		if len(IDs) != len(list) {
-			t.Fatalf("wrong number of IDs returned: want %d, got %d", len(IDs), len(list))
-		}
-
-		sort.Sort(IDs)
-		sort.Sort(list)
-
-		if !reflect.DeepEqual(IDs, list) {
-			t.Fatalf("lists aren't equal, want:\n  %v\n  got:\n%v\n", IDs, list)
-		}
-
-		var handles []backend.Handle
-		for _, ts := range testStrings {
-			id, err := restic.ParseID(ts.id)
 			test.OK(t, err)
 
-			h := backend.Handle{Type: tpe, Name: id.String()}
+			// test that the blob is gone
-
+			ok, err := beTest(context.TODO(), b, h)
-			found, err := beTest(context.TODO(), b, h)
 			test.OK(t, err)
-			test.Assert(t, found, fmt.Sprintf("id %v/%q not found", tpe, id))
+			test.Assert(t, !ok, "removed blob still present")
 
-			handles = append(handles, h)
+			// create blob
-		}
+			err = b.Save(context.TODO(), h, backend.NewByteReader([]byte(ts.data), b.Hasher()))
+			test.OK(t, err)
 
-		test.OK(t, s.delayedRemove(t, b, handles...))
+			// list items
+			IDs := restic.IDs{}
+
+			for _, ts := range testStrings {
+				id, err := restic.ParseID(ts.id)
+				test.OK(t, err)
+				IDs = append(IDs, id)
+			}
+
+			list := delayedList(t, b, tpe, len(IDs), s.WaitForDelayedRemoval)
+			if len(IDs) != len(list) {
+				t.Fatalf("wrong number of IDs returned: want %d, got %d", len(IDs), len(list))
+			}
+
+			sort.Sort(IDs)
+			sort.Sort(list)
+
+			if !reflect.DeepEqual(IDs, list) {
+				t.Fatalf("lists aren't equal, want:\n  %v\n  got:\n%v\n", IDs, list)
+			}
+
+			var handles []backend.Handle
+			for _, ts := range testStrings {
+				id, err := restic.ParseID(ts.id)
+				test.OK(t, err)
+
+				h := backend.Handle{Type: tpe, Name: id.String()}
+
+				found, err := beTest(context.TODO(), b, h)
+				test.OK(t, err)
+				test.Assert(t, found, fmt.Sprintf("id %v/%q not found", tpe, id))
+
+				handles = append(handles, h)
+			}
+
+			test.OK(t, s.delayedRemove(t, b, handles...))
+		})
 	}
 }