From e21ea66953d9ea8a63ff081781c4dec86679dbec Mon Sep 17 00:00:00 2001 From: Christian Kemper Date: Sat, 27 Feb 2016 10:58:45 -0800 Subject: [PATCH] Added support to s3.go to optionally use a local and sftp compatible repository layout. Updated the s3 config to use the s3 repository layout to keep backward compatibility with existing s3 repositories Updated the gcs config to use the new local compatible repository layout, allowing the use of "gsutil rsync" to sync content into Google Cloud Storage and transparently access the synced content using restic. Added support for using gs:http:// to enable the new repository layout on other s3 compatible cloud providers as well. --- src/cmds/restic/global.go | 2 +- src/restic/backend/gcs/config.go | 31 +++++++-- src/restic/backend/gcs/config_test.go | 96 ++++++++++++++++++++------- src/restic/backend/s3/config.go | 17 ++--- src/restic/backend/s3/s3.go | 48 ++++++++++++-- src/restic/location/location_test.go | 28 ++++---- 6 files changed, 166 insertions(+), 56 deletions(-) diff --git a/src/cmds/restic/global.go b/src/cmds/restic/global.go index 8f491a4a5..b0af07ee0 100644 --- a/src/cmds/restic/global.go +++ b/src/cmds/restic/global.go @@ -339,7 +339,7 @@ func open(s string) (restic.Backend, error) { cfg.Secret = os.Getenv("GS_SECRET_ACCESS_KEY") } debug.Log("open", "opening gcs repository at %#v", cfg) - be, err := s3.Open(cfg) + be, err = s3.Open(cfg) case "rest": be, err = rest.Open(loc.Config.(rest.Config)) default: diff --git a/src/restic/backend/gcs/config.go b/src/restic/backend/gcs/config.go index 811565dc7..52c57419e 100644 --- a/src/restic/backend/gcs/config.go +++ b/src/restic/backend/gcs/config.go @@ -2,6 +2,7 @@ package gcs import ( "errors" + "net/url" "restic/backend/s3" "strings" ) @@ -11,11 +12,31 @@ const gcsEndpoint = "storage.googleapis.com" const defaultPrefix = "restic" -// ParseConfig parses the string s and extracts the gcs config. The two -// supported configuration formats are gcs://bucketname/prefix and -// gcs:bucketname/prefix. +// ParseConfig parses the string s and extracts the s3 config. The two +// supported configuration formats are gs://bucketname/prefix and +// gs:bucketname/prefix. +// +// If no prefix is given the prefix "restic" will be used. func ParseConfig(s string) (interface{}, error) { + s = strings.TrimRight(s, "/") // remove trailing slashes switch { + case strings.HasPrefix(s, "gs:http"): + // assume that a URL has been specified, parse it and + // use the host as the endpoint and the path as the + // bucket name and prefix + url, err := url.Parse(s[3:]) + if err != nil { + return nil, err + } + + if url.Path == "" { + return nil, errors.New("gs: bucket name not found") + } + + path := strings.SplitN(url.Path[1:], "/", 2) + // create an s3 configuration using the local + // compatible repository layout + return s3.NewConfig(url.Host, path, url.Scheme == "http", true) case strings.HasPrefix(s, "gs://"): s = s[5:] case strings.HasPrefix(s, "gs:"): @@ -23,6 +44,8 @@ func ParseConfig(s string) (interface{}, error) { default: return nil, errors.New(`gcs: config does not start with "gs"`) } + // use the first entry of the path as bucket and the + // remainder as prefix p := strings.SplitN(s, "/", 2) - return s3.NewConfig(gcsEndpoint, p, false) + return s3.NewConfig(gcsEndpoint, p, false, true) } diff --git a/src/restic/backend/gcs/config_test.go b/src/restic/backend/gcs/config_test.go index c5843e371..c672c9c0c 100644 --- a/src/restic/backend/gcs/config_test.go +++ b/src/restic/backend/gcs/config_test.go @@ -10,44 +10,92 @@ var configTests = []struct { cfg s3.Config }{ {"gs://bucketname", s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "restic", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "restic", + LocalLayout: true, }}, {"gs://bucketname/", s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "restic", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "restic", + LocalLayout: true, }}, {"gs://bucketname/prefix/dir", s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "prefix/dir", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "prefix/dir", + LocalLayout: true, }}, {"gs://bucketname/prefix/dir/", s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "prefix/dir", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "prefix/dir", + LocalLayout: true, }}, {"gs:bucketname", s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "restic", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "restic", + LocalLayout: true, }}, {"gs:bucketname/", s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "restic", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "restic", + LocalLayout: true, }}, {"gs:bucketname/prefix/dir", s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "prefix/dir", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "prefix/dir", + LocalLayout: true, }}, {"gs:bucketname/prefix/dir/", s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "prefix/dir", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "prefix/dir", + LocalLayout: true, + }}, + {"gs:https://hostname:9999/foobar", s3.Config{ + Endpoint: "hostname:9999", + Bucket: "foobar", + Prefix: "restic", + LocalLayout: true, + }}, + {"gs:https://hostname:9999/foobar/", s3.Config{ + Endpoint: "hostname:9999", + Bucket: "foobar", + Prefix: "restic", + LocalLayout: true, + }}, + {"gs:http://hostname:9999/foobar", s3.Config{ + Endpoint: "hostname:9999", + Bucket: "foobar", + Prefix: "restic", + UseHTTP: true, + LocalLayout: true, + }}, + {"gs:http://hostname:9999/foobar/", s3.Config{ + Endpoint: "hostname:9999", + Bucket: "foobar", + Prefix: "restic", + UseHTTP: true, + LocalLayout: true, + }}, + {"gs:http://hostname:9999/bucket/prefix/directory", s3.Config{ + Endpoint: "hostname:9999", + Bucket: "bucket", + Prefix: "prefix/directory", + UseHTTP: true, + LocalLayout: true, + }}, + {"gs:http://hostname:9999/bucket/prefix/directory/", s3.Config{ + Endpoint: "hostname:9999", + Bucket: "bucket", + Prefix: "prefix/directory", + UseHTTP: true, + LocalLayout: true, }}, } diff --git a/src/restic/backend/s3/config.go b/src/restic/backend/s3/config.go index 927aa2026..408179df5 100644 --- a/src/restic/backend/s3/config.go +++ b/src/restic/backend/s3/config.go @@ -16,6 +16,7 @@ type Config struct { KeyID, Secret string Bucket string Prefix string + LocalLayout bool } const defaultPrefix = "restic" @@ -25,7 +26,6 @@ const defaultPrefix = "restic" // s3:host:bucketname/prefix. The host can also be a valid s3 region // name. If no prefix is given the prefix "restic" will be used. func ParseConfig(s string) (interface{}, error) { - s = strings.TrimRight(s, "/") // remove trailing slashes switch { case strings.HasPrefix(s, "s3:http"): // assume that a URL has been specified, parse it and @@ -41,7 +41,7 @@ func ParseConfig(s string) (interface{}, error) { } path := strings.SplitN(url.Path[1:], "/", 2) - return NewConfig(url.Host, path, url.Scheme == "http") + return NewConfig(url.Host, path, url.Scheme == "http", false) case strings.HasPrefix(s, "s3://"): s = s[5:] case strings.HasPrefix(s, "s3:"): @@ -52,13 +52,13 @@ func ParseConfig(s string) (interface{}, error) { // use the first entry of the path as the endpoint and the // remainder as bucket name and prefix path := strings.SplitN(s, "/", 3) - return NewConfig(path[0], path[1:], false) + return NewConfig(path[0], path[1:], false, false) } // NewConfig creates a Config at the specified endpoint. The Bucket is // the first entry in p and the remaining entries will be used as the // object name prefix. -func NewConfig(endpoint string, bucketPrefix []string, useHTTP bool) (interface{}, error) { +func NewConfig(endpoint string, bucketPrefix []string, useHTTP bool, localLayout bool) (interface{}, error) { var prefix string switch { case len(bucketPrefix) < 1: @@ -69,9 +69,10 @@ func NewConfig(endpoint string, bucketPrefix []string, useHTTP bool) (interface{ prefix = path.Clean(bucketPrefix[1]) } return Config{ - Endpoint: endpoint, - UseHTTP: useHTTP, - Bucket: bucketPrefix[0], - Prefix: prefix, + Endpoint: endpoint, + UseHTTP: useHTTP, + Bucket: bucketPrefix[0], + Prefix: prefix, + LocalLayout: localLayout, }, nil } diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index 56a6833cc..34206f79d 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -9,19 +9,21 @@ import ( "restic/errors" - "github.com/minio/minio-go" - + "restic/backend" "restic/debug" + + "github.com/minio/minio-go" ) const connLimit = 10 // s3 is a backend which stores the data on an S3 endpoint. type s3 struct { - client *minio.Client - connChan chan struct{} - bucketname string - prefix string + client *minio.Client + connChan chan struct{} + bucketname string + prefix string + localLayout bool } // Open opens the S3 backend at bucket and region. The bucket is created if it @@ -34,7 +36,7 @@ func Open(cfg Config) (restic.Backend, error) { return nil, errors.Wrap(err, "minio.New") } - be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix} + be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix, localLayout: cfg.LocalLayout} be.createConnections() found, err := client.BucketExists(cfg.Bucket) @@ -58,9 +60,41 @@ func (be *s3) s3path(t restic.FileType, name string) string { if t == restic.ConfigFile { return path.Join(be.prefix, string(t)) } + if be.localLayout == true { + // use the local compatible repository layout + return path.Join(be.prefix, dirname(t, name), name) + } + // use the standard s3 repository layout return path.Join(be.prefix, string(t), name) } +// Construct the directory name for a given FileType and file name using +// the path names defined in backend.Paths. +// +// This will also use the first two characters of the blob name as a +// directory prefix to partition blobs into smaller directories. This +// is similar to the way local and sftp repositories are handling +// blob filenames. +func dirname(t restic.FileType, name string) string { + var n string + switch t { + case restic.DataFile: + n = backend.Paths.Data + if len(name) > 2 { + n = path.Join(n, name[:2]) + } + case restic.SnapshotFile: + n = backend.Paths.Snapshots + case restic.IndexFile: + n = backend.Paths.Index + case restic.LockFile: + n = backend.Paths.Locks + case restic.KeyFile: + n = backend.Paths.Keys + } + return n +} + func (be *s3) createConnections() { be.connChan = make(chan struct{}, connLimit) for i := 0; i < connLimit; i++ { diff --git a/src/restic/location/location_test.go b/src/restic/location/location_test.go index 870ffdaad..1462e2194 100644 --- a/src/restic/location/location_test.go +++ b/src/restic/location/location_test.go @@ -56,30 +56,34 @@ var parseTests = []struct { }}}, {"gs://bucketname", Location{Scheme: "gs", Config: s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "restic", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "restic", + LocalLayout: true, }}, }, {"gs://bucketname/prefix", Location{Scheme: "gs", Config: s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "prefix", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "prefix", + LocalLayout: true, }}, }, {"gs:bucketname", Location{Scheme: "gs", Config: s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "restic", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "restic", + LocalLayout: true, }}, }, {"gs:bucketname/prefix", Location{Scheme: "gs", Config: s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "prefix", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "prefix", + LocalLayout: true, }}, }, {"s3://eu-central-1/bucketname", Location{Scheme: "s3",