diff --git a/src/cmds/restic/global.go b/src/cmds/restic/global.go index 8f491a4a5..b0af07ee0 100644 --- a/src/cmds/restic/global.go +++ b/src/cmds/restic/global.go @@ -339,7 +339,7 @@ func open(s string) (restic.Backend, error) { cfg.Secret = os.Getenv("GS_SECRET_ACCESS_KEY") } debug.Log("open", "opening gcs repository at %#v", cfg) - be, err := s3.Open(cfg) + be, err = s3.Open(cfg) case "rest": be, err = rest.Open(loc.Config.(rest.Config)) default: diff --git a/src/restic/backend/gcs/config.go b/src/restic/backend/gcs/config.go index 811565dc7..52c57419e 100644 --- a/src/restic/backend/gcs/config.go +++ b/src/restic/backend/gcs/config.go @@ -2,6 +2,7 @@ package gcs import ( "errors" + "net/url" "restic/backend/s3" "strings" ) @@ -11,11 +12,31 @@ const gcsEndpoint = "storage.googleapis.com" const defaultPrefix = "restic" -// ParseConfig parses the string s and extracts the gcs config. The two -// supported configuration formats are gcs://bucketname/prefix and -// gcs:bucketname/prefix. +// ParseConfig parses the string s and extracts the s3 config. The two +// supported configuration formats are gs://bucketname/prefix and +// gs:bucketname/prefix. +// +// If no prefix is given the prefix "restic" will be used. func ParseConfig(s string) (interface{}, error) { + s = strings.TrimRight(s, "/") // remove trailing slashes switch { + case strings.HasPrefix(s, "gs:http"): + // assume that a URL has been specified, parse it and + // use the host as the endpoint and the path as the + // bucket name and prefix + url, err := url.Parse(s[3:]) + if err != nil { + return nil, err + } + + if url.Path == "" { + return nil, errors.New("gs: bucket name not found") + } + + path := strings.SplitN(url.Path[1:], "/", 2) + // create an s3 configuration using the local + // compatible repository layout + return s3.NewConfig(url.Host, path, url.Scheme == "http", true) case strings.HasPrefix(s, "gs://"): s = s[5:] case strings.HasPrefix(s, "gs:"): @@ -23,6 +44,8 @@ func ParseConfig(s string) (interface{}, error) { default: return nil, errors.New(`gcs: config does not start with "gs"`) } + // use the first entry of the path as bucket and the + // remainder as prefix p := strings.SplitN(s, "/", 2) - return s3.NewConfig(gcsEndpoint, p, false) + return s3.NewConfig(gcsEndpoint, p, false, true) } diff --git a/src/restic/backend/gcs/config_test.go b/src/restic/backend/gcs/config_test.go index c5843e371..c672c9c0c 100644 --- a/src/restic/backend/gcs/config_test.go +++ b/src/restic/backend/gcs/config_test.go @@ -10,44 +10,92 @@ var configTests = []struct { cfg s3.Config }{ {"gs://bucketname", s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "restic", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "restic", + LocalLayout: true, }}, {"gs://bucketname/", s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "restic", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "restic", + LocalLayout: true, }}, {"gs://bucketname/prefix/dir", s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "prefix/dir", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "prefix/dir", + LocalLayout: true, }}, {"gs://bucketname/prefix/dir/", s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "prefix/dir", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "prefix/dir", + LocalLayout: true, }}, {"gs:bucketname", s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "restic", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "restic", + LocalLayout: true, }}, {"gs:bucketname/", s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "restic", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "restic", + LocalLayout: true, }}, {"gs:bucketname/prefix/dir", s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "prefix/dir", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "prefix/dir", + LocalLayout: true, }}, {"gs:bucketname/prefix/dir/", s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "prefix/dir", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "prefix/dir", + LocalLayout: true, + }}, + {"gs:https://hostname:9999/foobar", s3.Config{ + Endpoint: "hostname:9999", + Bucket: "foobar", + Prefix: "restic", + LocalLayout: true, + }}, + {"gs:https://hostname:9999/foobar/", s3.Config{ + Endpoint: "hostname:9999", + Bucket: "foobar", + Prefix: "restic", + LocalLayout: true, + }}, + {"gs:http://hostname:9999/foobar", s3.Config{ + Endpoint: "hostname:9999", + Bucket: "foobar", + Prefix: "restic", + UseHTTP: true, + LocalLayout: true, + }}, + {"gs:http://hostname:9999/foobar/", s3.Config{ + Endpoint: "hostname:9999", + Bucket: "foobar", + Prefix: "restic", + UseHTTP: true, + LocalLayout: true, + }}, + {"gs:http://hostname:9999/bucket/prefix/directory", s3.Config{ + Endpoint: "hostname:9999", + Bucket: "bucket", + Prefix: "prefix/directory", + UseHTTP: true, + LocalLayout: true, + }}, + {"gs:http://hostname:9999/bucket/prefix/directory/", s3.Config{ + Endpoint: "hostname:9999", + Bucket: "bucket", + Prefix: "prefix/directory", + UseHTTP: true, + LocalLayout: true, }}, } diff --git a/src/restic/backend/s3/config.go b/src/restic/backend/s3/config.go index 927aa2026..408179df5 100644 --- a/src/restic/backend/s3/config.go +++ b/src/restic/backend/s3/config.go @@ -16,6 +16,7 @@ type Config struct { KeyID, Secret string Bucket string Prefix string + LocalLayout bool } const defaultPrefix = "restic" @@ -25,7 +26,6 @@ const defaultPrefix = "restic" // s3:host:bucketname/prefix. The host can also be a valid s3 region // name. If no prefix is given the prefix "restic" will be used. func ParseConfig(s string) (interface{}, error) { - s = strings.TrimRight(s, "/") // remove trailing slashes switch { case strings.HasPrefix(s, "s3:http"): // assume that a URL has been specified, parse it and @@ -41,7 +41,7 @@ func ParseConfig(s string) (interface{}, error) { } path := strings.SplitN(url.Path[1:], "/", 2) - return NewConfig(url.Host, path, url.Scheme == "http") + return NewConfig(url.Host, path, url.Scheme == "http", false) case strings.HasPrefix(s, "s3://"): s = s[5:] case strings.HasPrefix(s, "s3:"): @@ -52,13 +52,13 @@ func ParseConfig(s string) (interface{}, error) { // use the first entry of the path as the endpoint and the // remainder as bucket name and prefix path := strings.SplitN(s, "/", 3) - return NewConfig(path[0], path[1:], false) + return NewConfig(path[0], path[1:], false, false) } // NewConfig creates a Config at the specified endpoint. The Bucket is // the first entry in p and the remaining entries will be used as the // object name prefix. -func NewConfig(endpoint string, bucketPrefix []string, useHTTP bool) (interface{}, error) { +func NewConfig(endpoint string, bucketPrefix []string, useHTTP bool, localLayout bool) (interface{}, error) { var prefix string switch { case len(bucketPrefix) < 1: @@ -69,9 +69,10 @@ func NewConfig(endpoint string, bucketPrefix []string, useHTTP bool) (interface{ prefix = path.Clean(bucketPrefix[1]) } return Config{ - Endpoint: endpoint, - UseHTTP: useHTTP, - Bucket: bucketPrefix[0], - Prefix: prefix, + Endpoint: endpoint, + UseHTTP: useHTTP, + Bucket: bucketPrefix[0], + Prefix: prefix, + LocalLayout: localLayout, }, nil } diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index 56a6833cc..34206f79d 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -9,19 +9,21 @@ import ( "restic/errors" - "github.com/minio/minio-go" - + "restic/backend" "restic/debug" + + "github.com/minio/minio-go" ) const connLimit = 10 // s3 is a backend which stores the data on an S3 endpoint. type s3 struct { - client *minio.Client - connChan chan struct{} - bucketname string - prefix string + client *minio.Client + connChan chan struct{} + bucketname string + prefix string + localLayout bool } // Open opens the S3 backend at bucket and region. The bucket is created if it @@ -34,7 +36,7 @@ func Open(cfg Config) (restic.Backend, error) { return nil, errors.Wrap(err, "minio.New") } - be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix} + be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix, localLayout: cfg.LocalLayout} be.createConnections() found, err := client.BucketExists(cfg.Bucket) @@ -58,9 +60,41 @@ func (be *s3) s3path(t restic.FileType, name string) string { if t == restic.ConfigFile { return path.Join(be.prefix, string(t)) } + if be.localLayout == true { + // use the local compatible repository layout + return path.Join(be.prefix, dirname(t, name), name) + } + // use the standard s3 repository layout return path.Join(be.prefix, string(t), name) } +// Construct the directory name for a given FileType and file name using +// the path names defined in backend.Paths. +// +// This will also use the first two characters of the blob name as a +// directory prefix to partition blobs into smaller directories. This +// is similar to the way local and sftp repositories are handling +// blob filenames. +func dirname(t restic.FileType, name string) string { + var n string + switch t { + case restic.DataFile: + n = backend.Paths.Data + if len(name) > 2 { + n = path.Join(n, name[:2]) + } + case restic.SnapshotFile: + n = backend.Paths.Snapshots + case restic.IndexFile: + n = backend.Paths.Index + case restic.LockFile: + n = backend.Paths.Locks + case restic.KeyFile: + n = backend.Paths.Keys + } + return n +} + func (be *s3) createConnections() { be.connChan = make(chan struct{}, connLimit) for i := 0; i < connLimit; i++ { diff --git a/src/restic/location/location_test.go b/src/restic/location/location_test.go index 870ffdaad..1462e2194 100644 --- a/src/restic/location/location_test.go +++ b/src/restic/location/location_test.go @@ -56,30 +56,34 @@ var parseTests = []struct { }}}, {"gs://bucketname", Location{Scheme: "gs", Config: s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "restic", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "restic", + LocalLayout: true, }}, }, {"gs://bucketname/prefix", Location{Scheme: "gs", Config: s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "prefix", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "prefix", + LocalLayout: true, }}, }, {"gs:bucketname", Location{Scheme: "gs", Config: s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "restic", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "restic", + LocalLayout: true, }}, }, {"gs:bucketname/prefix", Location{Scheme: "gs", Config: s3.Config{ - Endpoint: "storage.googleapis.com", - Bucket: "bucketname", - Prefix: "prefix", + Endpoint: "storage.googleapis.com", + Bucket: "bucketname", + Prefix: "prefix", + LocalLayout: true, }}, }, {"s3://eu-central-1/bucketname", Location{Scheme: "s3",