From 3541d06d076baa7ccc04a8e8576971e1429a0e73 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sun, 16 Jul 2017 20:16:02 +0200
Subject: [PATCH] repo: Split packers for tree and data

The code now bundles tree blobs and data blobs into different pack
files, so we'll end up with pack files that either only contain data or
trees. This is in preparation to adding a cache (#1040), because
tree-only pack files can easily be cached later on.
---
 internal/repository/packer_manager.go      |  4 +-
 internal/repository/packer_manager_test.go |  2 +-
 internal/repository/repository.go          | 53 +++++++++++++++-------
 3 files changed, 39 insertions(+), 20 deletions(-)

diff --git a/internal/repository/packer_manager.go b/internal/repository/packer_manager.go
index 3b2734f5..aa1885cc 100644
--- a/internal/repository/packer_manager.go
+++ b/internal/repository/packer_manager.go
@@ -50,7 +50,7 @@ func newPackerManager(be Saver, key *crypto.Key) *packerManager {
 
 // findPacker returns a packer for a new blob of size bytes. Either a new one is
 // created or one is returned that already has some blobs.
-func (r *packerManager) findPacker(size uint) (packer *Packer, err error) {
+func (r *packerManager) findPacker() (packer *Packer, err error) {
 	r.pm.Lock()
 	defer r.pm.Unlock()
 
@@ -62,7 +62,7 @@ func (r *packerManager) findPacker(size uint) (packer *Packer, err error) {
 	}
 
 	// no suitable packer found, return new
-	debug.Log("create new pack for %d bytes", size)
+	debug.Log("create new pack")
 	tmpfile, err := fs.TempFile("", "restic-temp-pack-")
 	if err != nil {
 		return nil, errors.Wrap(err, "fs.TempFile")
diff --git a/internal/repository/packer_manager_test.go b/internal/repository/packer_manager_test.go
index efdd7ff4..aface097 100644
--- a/internal/repository/packer_manager_test.go
+++ b/internal/repository/packer_manager_test.go
@@ -72,7 +72,7 @@ func fillPacks(t testing.TB, rnd *randReader, be Saver, pm *packerManager, buf [
 		l := rnd.rand.Intn(1 << 20)
 		seed := rnd.rand.Int63()
 
-		packer, err := pm.findPacker(uint(l))
+		packer, err := pm.findPacker()
 		if err != nil {
 			t.Fatal(err)
 		}
diff --git a/internal/repository/repository.go b/internal/repository/repository.go
index 7623cde8..ae35f78c 100644
--- a/internal/repository/repository.go
+++ b/internal/repository/repository.go
@@ -24,15 +24,17 @@ type Repository struct {
 	keyName string
 	idx     *MasterIndex
 
-	*packerManager
+	treePM *packerManager
+	dataPM *packerManager
 }
 
 // New returns a new repository with backend be.
 func New(be restic.Backend) *Repository {
 	repo := &Repository{
-		be:            be,
-		idx:           NewMasterIndex(),
-		packerManager: newPackerManager(be, nil),
+		be:     be,
+		idx:    NewMasterIndex(),
+		dataPM: newPackerManager(be, nil),
+		treePM: newPackerManager(be, nil),
 	}
 
 	return repo
@@ -180,7 +182,18 @@ func (r *Repository) SaveAndEncrypt(ctx context.Context, t restic.BlobType, data
 	}
 
 	// find suitable packer and add blob
-	packer, err := r.findPacker(uint(len(ciphertext)))
+	var pm *packerManager
+
+	switch t {
+	case restic.TreeBlob:
+		pm = r.treePM
+	case restic.DataBlob:
+		pm = r.dataPM
+	default:
+		panic(fmt.Sprintf("invalid type: %v", t))
+	}
+
+	packer, err := pm.findPacker()
 	if err != nil {
 		return restic.ID{}, err
 	}
@@ -194,7 +207,7 @@ func (r *Repository) SaveAndEncrypt(ctx context.Context, t restic.BlobType, data
 	// if the pack is not full enough, put back to the list
 	if packer.Size() < minPackSize {
 		debug.Log("pack is not full enough (%d bytes)", packer.Size())
-		r.insertPacker(packer)
+		pm.insertPacker(packer)
 		return *id, nil
 	}
 
@@ -238,18 +251,22 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by
 
 // Flush saves all remaining packs.
 func (r *Repository) Flush() error {
-	r.pm.Lock()
-	defer r.pm.Unlock()
+	for _, pm := range []*packerManager{r.dataPM, r.treePM} {
+		pm.pm.Lock()
 
-	debug.Log("manually flushing %d packs", len(r.packerManager.packers))
-
-	for _, p := range r.packerManager.packers {
-		err := r.savePacker(p)
-		if err != nil {
-			return err
+		debug.Log("manually flushing %d packs", len(pm.packers))
+		for _, p := range pm.packers {
+			err := r.savePacker(p)
+			if err != nil {
+				pm.pm.Unlock()
+				return err
+			}
 		}
+		pm.packers = pm.packers[:0]
+
+		pm.pm.Unlock()
 	}
-	r.packerManager.packers = r.packerManager.packers[:0]
+
 	return nil
 }
 
@@ -371,7 +388,8 @@ func (r *Repository) SearchKey(ctx context.Context, password string, maxKeys int
 	}
 
 	r.key = key.master
-	r.packerManager.key = key.master
+	r.dataPM.key = key.master
+	r.treePM.key = key.master
 	r.keyName = key.Name()
 	r.cfg, err = restic.LoadConfig(ctx, r)
 	return err
@@ -405,7 +423,8 @@ func (r *Repository) init(ctx context.Context, password string, cfg restic.Confi
 	}
 
 	r.key = key.master
-	r.packerManager.key = key.master
+	r.dataPM.key = key.master
+	r.treePM.key = key.master
 	r.keyName = key.Name()
 	r.cfg = cfg
 	_, err = r.SaveJSONUnpacked(ctx, restic.ConfigFile, cfg)