Added support for a configurable directory prefix for blob names.

Extracted a Filename and Dirname function and used it in the s3 and
sftp backends. Added support for setting the length of the directory
prefix in the s3 configuration. Introduced a NewConfig method for
creating the s3 configuration.

src/restic/backend/paths.go

@@ -1,6 +1,10 @@
 package backend
 
-import "os"
+import (
+	"os"
+	"path"
+	"restic"
+)
 
 // Paths contains the default paths for file-based backends (e.g. local).
 var Paths = struct {
@@ -24,3 +28,53 @@ var Paths = struct {
 // Modes holds the default modes for directories and files for file-based
 // backends.
 var Modes = struct{ Dir, File os.FileMode }{0700, 0600}
+
+// Join joins the given paths and cleans them afterwards. This always uses
+// forward slashes, which is required by all non local backends.
+func Join(parts ...string) string {
+	return path.Clean(path.Join(parts...))
+}
+
+// Filename constructs the path for given base name restic.Type and
+// name.
+//
+// This can optionally use a number of characters of the blob name as
+// a directory prefix to partition blobs into smaller directories.
+//
+// Currently local and sftp repositories are handling blob filenames
+// using a dirPrefixLen of 2.
+func Filename(base string, t restic.FileType, name string, dirPrefixLen int) string {
+	if t == restic.ConfigFile {
+		return Join(base, Paths.Config)
+	}
+
+	return Join(base, Dirname(t, name, dirPrefixLen), name)
+}
+
+// Dirname constructs the directory name for a given FileType and file
+// name using the path names defined in Paths.
+//
+// This can optionally use a number of characters of the blob name as
+// a directory prefix to partition blobs into smaller directories.
+//
+// Currently local and sftp repositories are handling blob filenames
+// using a dirPrefixLen of 2.
+func Dirname(t restic.FileType, name string, dirPrefixLen int) string {
+	var n string
+	switch t {
+	case restic.DataFile:
+		n = Paths.Data
+		if dirPrefixLen > 0 && len(name) > dirPrefixLen {
+			n = path.Join(n, name[:dirPrefixLen])
+		}
+	case restic.SnapshotFile:
+		n = Paths.Snapshots
+	case restic.IndexFile:
+		n = Paths.Index
+	case restic.LockFile:
+		n = Paths.Locks
+	case restic.KeyFile:
+		n = Paths.Keys
+	}
+	return n
+}

src/restic/backend/paths_test.go

@@ -0,0 +1,55 @@
+package backend_test
+
+import (
+	"restic"
+	"testing"
+
+	"restic/backend"
+)
+
+var pathsTests = []struct {
+	base      string
+	t         restic.FileType
+	name      string
+	prefixLen int
+	res       string
+}{
+	{"base", restic.DataFile, "abcdef", 0, "base/data/abcdef"},
+	{"base", restic.DataFile, "abcdef", 2, "base/data/ab/abcdef"},
+	{"base", restic.DataFile, "abcdef", 4, "base/data/abcd/abcdef"},
+	{"base", restic.DataFile, "ab", 2, "base/data/ab"},
+	{"base", restic.DataFile, "abcd", 4, "base/data/abcd"},
+
+	{"", restic.DataFile, "abcdef", 0, "data/abcdef"},
+	{"", restic.DataFile, "abcdef", 2, "data/ab/abcdef"},
+	{"", restic.DataFile, "abcdef", 4, "data/abcd/abcdef"},
+	{"", restic.DataFile, "ab", 2, "data/ab"},
+	{"", restic.DataFile, "abcd", 4, "data/abcd"},
+
+	{"base", restic.ConfigFile, "file", 0, "base/config"},
+	{"", restic.ConfigFile, "file", 0, "config"},
+
+	{"base", restic.SnapshotFile, "file", 0, "base/snapshots/file"},
+	{"base", restic.SnapshotFile, "file", 2, "base/snapshots/file"},
+
+	{"base", restic.IndexFile, "file", 0, "base/index/file"},
+	{"base", restic.IndexFile, "file", 2, "base/index/file"},
+
+	{"base", restic.LockFile, "file", 0, "base/locks/file"},
+	{"base", restic.LockFile, "file", 2, "base/locks/file"},
+
+	{"base", restic.KeyFile, "file", 0, "base/keys/file"},
+	{"base", restic.KeyFile, "file", 2, "base/keys/file"},
+}
+
+func TestFilename(t *testing.T) {
+	for i, test := range pathsTests {
+		res := backend.Filename(test.base, test.t, test.name, test.prefixLen)
+
+		if test.res != res {
+			t.Errorf("test %d: result does not match, want %q, got %q",
+				i, test.res, res)
+		}
+	}
+
+}

src/restic/backend/s3/config.go

@@ -16,9 +16,13 @@ type Config struct {
 	KeyID, Secret string
 	Bucket        string
 	Prefix        string
+	DirPrefixLen  int
 }
 
-const defaultPrefix = "restic"
+const (
+	defaultPrefix       = "restic"
+	defaultDirPrefixLen = 0
+)
 
 // ParseConfig parses the string s and extracts the s3 config. The two
 // supported configuration formats are s3://host/bucketname/prefix and
@@ -40,7 +44,7 @@ func ParseConfig(s string) (interface{}, error) {
 		}
 
 		path := strings.SplitN(url.Path[1:], "/", 2)
-		return createConfig(url.Host, path, url.Scheme == "http")
+		return NewConfig(url.Host, path, url.Scheme == "http", defaultDirPrefixLen)
 	case strings.HasPrefix(s, "s3://"):
 		s = s[5:]
 	case strings.HasPrefix(s, "s3:"):
@@ -51,10 +55,13 @@ func ParseConfig(s string) (interface{}, error) {
 	// use the first entry of the path as the endpoint and the
 	// remainder as bucket name and prefix
 	path := strings.SplitN(s, "/", 3)
-	return createConfig(path[0], path[1:], false)
+	return NewConfig(path[0], path[1:], false, defaultDirPrefixLen)
 }
 
-func createConfig(endpoint string, p []string, useHTTP bool) (interface{}, error) {
+// NewConfig creates a Config at the specified endpoint. The Bucket is
+// the first entry in path and the remaining entries will be used as the
+// object name prefix.
+func NewConfig(endpoint string, p []string, useHTTP bool, dirPrefixLen int) (interface{}, error) {
 	var prefix string
 	switch {
 	case len(p) < 1:
@@ -65,9 +72,10 @@ func createConfig(endpoint string, p []string, useHTTP bool) (interface{}, error
 		prefix = path.Clean(p[1])
 	}
 	return Config{
-		Endpoint: endpoint,
-		UseHTTP:  useHTTP,
-		Bucket:   p[0],
-		Prefix:   prefix,
+		Endpoint:     endpoint,
+		UseHTTP:      useHTTP,
+		Bucket:       p[0],
+		Prefix:       prefix,
+		DirPrefixLen: dirPrefixLen,
 	}, nil
 }

src/restic/backend/s3/s3.go

@@ -3,25 +3,25 @@ package s3
 import (
 	"bytes"
 	"io"
-	"path"
 	"restic"
 	"strings"
 
+	"restic/backend"
+	"restic/debug"
 	"restic/errors"
 
 	"github.com/minio/minio-go"
-
-	"restic/debug"
 )
 
 const connLimit = 10
 
 // s3 is a backend which stores the data on an S3 endpoint.
 type s3 struct {
-	client     *minio.Client
-	connChan   chan struct{}
-	bucketname string
-	prefix     string
+	client       *minio.Client
+	connChan     chan struct{}
+	bucketname   string
+	prefix       string
+	dirPrefixLen int
 }
 
 // Open opens the S3 backend at bucket and region. The bucket is created if it
@@ -34,7 +34,7 @@ func Open(cfg Config) (restic.Backend, error) {
 		return nil, errors.Wrap(err, "minio.New")
 	}
 
-	be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix}
+	be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix, dirPrefixLen: cfg.DirPrefixLen}
 	be.createConnections()
 
 	found, err := client.BucketExists(cfg.Bucket)
@@ -55,10 +55,7 @@ func Open(cfg Config) (restic.Backend, error) {
 }
 
 func (be *s3) s3path(t restic.FileType, name string) string {
-	if t == restic.ConfigFile {
-		return path.Join(be.prefix, string(t))
-	}
-	return path.Join(be.prefix, string(t), name)
+	return backend.Filename(be.prefix, t, name, be.dirPrefixLen)
 }
 
 func (be *s3) createConnections() {

src/restic/backend/sftp/sftp.go

@@ -23,6 +23,7 @@ import (
 
 const (
 	tempfileRandomSuffixLength = 10
+	defaultDirPrefixLen        = 2
 )
 
 // SFTP is a backend in a directory accessed via SFTP.
@@ -298,32 +299,12 @@ func Join(parts ...string) string {
 
 // Construct path for given restic.Type and name.
 func (r *SFTP) filename(t restic.FileType, name string) string {
-	if t == restic.ConfigFile {
-		return Join(r.p, "config")
-	}
-
-	return Join(r.dirname(t, name), name)
+	return backend.Filename(r.p, t, name, defaultDirPrefixLen)
 }
 
 // Construct directory for given backend.Type.
 func (r *SFTP) dirname(t restic.FileType, name string) string {
-	var n string
-	switch t {
-	case restic.DataFile:
-		n = backend.Paths.Data
-		if len(name) > 2 {
-			n = Join(n, name[:2])
-		}
-	case restic.SnapshotFile:
-		n = backend.Paths.Snapshots
-	case restic.IndexFile:
-		n = backend.Paths.Index
-	case restic.LockFile:
-		n = backend.Paths.Locks
-	case restic.KeyFile:
-		n = backend.Paths.Keys
-	}
-	return Join(r.p, n)
+	return Join(r.p, backend.Dirname(t, name, defaultDirPrefixLen))
 }
 
 // Load returns the data stored in the backend for h at the given offset