From 03209b95fd05b2b1928732c6d765dda45fe26a9e Mon Sep 17 00:00:00 2001 From: Christian Kemper Date: Fri, 30 Dec 2016 14:59:34 -0800 Subject: [PATCH] Added support for a configurable directory prefix for blob names. Extracted a Filename and Dirname function and used it in the s3 and sftp backends. Added support for setting the length of the directory prefix in the s3 configuration. Introduced a NewConfig method for creating the s3 configuration. --- src/restic/backend/paths.go | 56 +++++++++++++++++++++++++++++++- src/restic/backend/paths_test.go | 55 +++++++++++++++++++++++++++++++ src/restic/backend/s3/config.go | 24 +++++++++----- src/restic/backend/s3/s3.go | 21 +++++------- src/restic/backend/sftp/sftp.go | 25 ++------------ 5 files changed, 138 insertions(+), 43 deletions(-) create mode 100644 src/restic/backend/paths_test.go diff --git a/src/restic/backend/paths.go b/src/restic/backend/paths.go index 940e9fcb9..6804058ec 100644 --- a/src/restic/backend/paths.go +++ b/src/restic/backend/paths.go @@ -1,6 +1,10 @@ package backend -import "os" +import ( + "os" + "path" + "restic" +) // Paths contains the default paths for file-based backends (e.g. local). var Paths = struct { @@ -24,3 +28,53 @@ var Paths = struct { // Modes holds the default modes for directories and files for file-based // backends. var Modes = struct{ Dir, File os.FileMode }{0700, 0600} + +// Join joins the given paths and cleans them afterwards. This always uses +// forward slashes, which is required by all non local backends. +func Join(parts ...string) string { + return path.Clean(path.Join(parts...)) +} + +// Filename constructs the path for given base name restic.Type and +// name. +// +// This can optionally use a number of characters of the blob name as +// a directory prefix to partition blobs into smaller directories. +// +// Currently local and sftp repositories are handling blob filenames +// using a dirPrefixLen of 2. +func Filename(base string, t restic.FileType, name string, dirPrefixLen int) string { + if t == restic.ConfigFile { + return Join(base, Paths.Config) + } + + return Join(base, Dirname(t, name, dirPrefixLen), name) +} + +// Dirname constructs the directory name for a given FileType and file +// name using the path names defined in Paths. +// +// This can optionally use a number of characters of the blob name as +// a directory prefix to partition blobs into smaller directories. +// +// Currently local and sftp repositories are handling blob filenames +// using a dirPrefixLen of 2. +func Dirname(t restic.FileType, name string, dirPrefixLen int) string { + var n string + switch t { + case restic.DataFile: + n = Paths.Data + if dirPrefixLen > 0 && len(name) > dirPrefixLen { + n = path.Join(n, name[:dirPrefixLen]) + } + case restic.SnapshotFile: + n = Paths.Snapshots + case restic.IndexFile: + n = Paths.Index + case restic.LockFile: + n = Paths.Locks + case restic.KeyFile: + n = Paths.Keys + } + return n +} diff --git a/src/restic/backend/paths_test.go b/src/restic/backend/paths_test.go new file mode 100644 index 000000000..55e49d64b --- /dev/null +++ b/src/restic/backend/paths_test.go @@ -0,0 +1,55 @@ +package backend_test + +import ( + "restic" + "testing" + + "restic/backend" +) + +var pathsTests = []struct { + base string + t restic.FileType + name string + prefixLen int + res string +}{ + {"base", restic.DataFile, "abcdef", 0, "base/data/abcdef"}, + {"base", restic.DataFile, "abcdef", 2, "base/data/ab/abcdef"}, + {"base", restic.DataFile, "abcdef", 4, "base/data/abcd/abcdef"}, + {"base", restic.DataFile, "ab", 2, "base/data/ab"}, + {"base", restic.DataFile, "abcd", 4, "base/data/abcd"}, + + {"", restic.DataFile, "abcdef", 0, "data/abcdef"}, + {"", restic.DataFile, "abcdef", 2, "data/ab/abcdef"}, + {"", restic.DataFile, "abcdef", 4, "data/abcd/abcdef"}, + {"", restic.DataFile, "ab", 2, "data/ab"}, + {"", restic.DataFile, "abcd", 4, "data/abcd"}, + + {"base", restic.ConfigFile, "file", 0, "base/config"}, + {"", restic.ConfigFile, "file", 0, "config"}, + + {"base", restic.SnapshotFile, "file", 0, "base/snapshots/file"}, + {"base", restic.SnapshotFile, "file", 2, "base/snapshots/file"}, + + {"base", restic.IndexFile, "file", 0, "base/index/file"}, + {"base", restic.IndexFile, "file", 2, "base/index/file"}, + + {"base", restic.LockFile, "file", 0, "base/locks/file"}, + {"base", restic.LockFile, "file", 2, "base/locks/file"}, + + {"base", restic.KeyFile, "file", 0, "base/keys/file"}, + {"base", restic.KeyFile, "file", 2, "base/keys/file"}, +} + +func TestFilename(t *testing.T) { + for i, test := range pathsTests { + res := backend.Filename(test.base, test.t, test.name, test.prefixLen) + + if test.res != res { + t.Errorf("test %d: result does not match, want %q, got %q", + i, test.res, res) + } + } + +} diff --git a/src/restic/backend/s3/config.go b/src/restic/backend/s3/config.go index 2df02b58c..51ae996fb 100644 --- a/src/restic/backend/s3/config.go +++ b/src/restic/backend/s3/config.go @@ -16,9 +16,13 @@ type Config struct { KeyID, Secret string Bucket string Prefix string + DirPrefixLen int } -const defaultPrefix = "restic" +const ( + defaultPrefix = "restic" + defaultDirPrefixLen = 0 +) // ParseConfig parses the string s and extracts the s3 config. The two // supported configuration formats are s3://host/bucketname/prefix and @@ -40,7 +44,7 @@ func ParseConfig(s string) (interface{}, error) { } path := strings.SplitN(url.Path[1:], "/", 2) - return createConfig(url.Host, path, url.Scheme == "http") + return NewConfig(url.Host, path, url.Scheme == "http", defaultDirPrefixLen) case strings.HasPrefix(s, "s3://"): s = s[5:] case strings.HasPrefix(s, "s3:"): @@ -51,10 +55,13 @@ func ParseConfig(s string) (interface{}, error) { // use the first entry of the path as the endpoint and the // remainder as bucket name and prefix path := strings.SplitN(s, "/", 3) - return createConfig(path[0], path[1:], false) + return NewConfig(path[0], path[1:], false, defaultDirPrefixLen) } -func createConfig(endpoint string, p []string, useHTTP bool) (interface{}, error) { +// NewConfig creates a Config at the specified endpoint. The Bucket is +// the first entry in path and the remaining entries will be used as the +// object name prefix. +func NewConfig(endpoint string, p []string, useHTTP bool, dirPrefixLen int) (interface{}, error) { var prefix string switch { case len(p) < 1: @@ -65,9 +72,10 @@ func createConfig(endpoint string, p []string, useHTTP bool) (interface{}, error prefix = path.Clean(p[1]) } return Config{ - Endpoint: endpoint, - UseHTTP: useHTTP, - Bucket: p[0], - Prefix: prefix, + Endpoint: endpoint, + UseHTTP: useHTTP, + Bucket: p[0], + Prefix: prefix, + DirPrefixLen: dirPrefixLen, }, nil } diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index 56a6833cc..94cfbb31b 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -3,25 +3,25 @@ package s3 import ( "bytes" "io" - "path" "restic" "strings" + "restic/backend" + "restic/debug" "restic/errors" "github.com/minio/minio-go" - - "restic/debug" ) const connLimit = 10 // s3 is a backend which stores the data on an S3 endpoint. type s3 struct { - client *minio.Client - connChan chan struct{} - bucketname string - prefix string + client *minio.Client + connChan chan struct{} + bucketname string + prefix string + dirPrefixLen int } // Open opens the S3 backend at bucket and region. The bucket is created if it @@ -34,7 +34,7 @@ func Open(cfg Config) (restic.Backend, error) { return nil, errors.Wrap(err, "minio.New") } - be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix} + be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix, dirPrefixLen: cfg.DirPrefixLen} be.createConnections() found, err := client.BucketExists(cfg.Bucket) @@ -55,10 +55,7 @@ func Open(cfg Config) (restic.Backend, error) { } func (be *s3) s3path(t restic.FileType, name string) string { - if t == restic.ConfigFile { - return path.Join(be.prefix, string(t)) - } - return path.Join(be.prefix, string(t), name) + return backend.Filename(be.prefix, t, name, be.dirPrefixLen) } func (be *s3) createConnections() { diff --git a/src/restic/backend/sftp/sftp.go b/src/restic/backend/sftp/sftp.go index 95555441f..d8de18ce7 100644 --- a/src/restic/backend/sftp/sftp.go +++ b/src/restic/backend/sftp/sftp.go @@ -23,6 +23,7 @@ import ( const ( tempfileRandomSuffixLength = 10 + defaultDirPrefixLen = 2 ) // SFTP is a backend in a directory accessed via SFTP. @@ -298,32 +299,12 @@ func Join(parts ...string) string { // Construct path for given restic.Type and name. func (r *SFTP) filename(t restic.FileType, name string) string { - if t == restic.ConfigFile { - return Join(r.p, "config") - } - - return Join(r.dirname(t, name), name) + return backend.Filename(r.p, t, name, defaultDirPrefixLen) } // Construct directory for given backend.Type. func (r *SFTP) dirname(t restic.FileType, name string) string { - var n string - switch t { - case restic.DataFile: - n = backend.Paths.Data - if len(name) > 2 { - n = Join(n, name[:2]) - } - case restic.SnapshotFile: - n = backend.Paths.Snapshots - case restic.IndexFile: - n = backend.Paths.Index - case restic.LockFile: - n = backend.Paths.Locks - case restic.KeyFile: - n = backend.Paths.Keys - } - return Join(r.p, n) + return Join(r.p, backend.Dirname(t, name, defaultDirPrefixLen)) } // Load returns the data stored in the backend for h at the given offset