Mirrors/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2025-03-16 00:00:05 +01:00
Code Issues Releases Wiki Activity

Added support to s3.go to optionally use a local and sftp compatible

Browse source 
repository layout.

Updated the s3 config to use the s3 repository layout to keep backward
compatibility with existing s3 repositories

Updated the gcs config to use the new local compatible repository
layout, allowing the use of "gsutil rsync" to sync content into Google
Cloud Storage and transparently access the synced content using
restic.

Added support for using gs:http:// to enable the new repository layout
on other s3 compatible cloud providers as well.
This commit is contained in:
Christian Kemper 2016-02-27 10:58:45 -08:00
parent 13cbd11818
commit e21ea66953
6 changed files with 166 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/cmds/restic/global.go
View file

@ -339,7 +339,7 @@ func open(s string) (restic.Backend, error) {
cfg.Secret = os.Getenv("GS_SECRET_ACCESS_KEY")
}
debug.Log("open", "opening gcs repository at %#v", cfg)
be, err := s3.Open(cfg)
be, err = s3.Open(cfg)
case "rest":
be, err = rest.Open(loc.Config.(rest.Config))
default:

31
src/restic/backend/gcs/config.go
View file

@ -2,6 +2,7 @@ package gcs
import (
"errors"
"net/url"
"restic/backend/s3"
"strings"
)
@ -11,11 +12,31 @@ const gcsEndpoint = "storage.googleapis.com"
const defaultPrefix = "restic"
// ParseConfig parses the string s and extracts the gcs config. The two
// supported configuration formats are gcs://bucketname/prefix and
// gcs:bucketname/prefix.
// ParseConfig parses the string s and extracts the s3 config. The two
// supported configuration formats are gs://bucketname/prefix and
// gs:bucketname/prefix.
//
// If no prefix is given the prefix "restic" will be used.
func ParseConfig(s string) (interface{}, error) {
s = strings.TrimRight(s, "/") // remove trailing slashes
switch {
case strings.HasPrefix(s, "gs:http"):
// assume that a URL has been specified, parse it and
// use the host as the endpoint and the path as the
// bucket name and prefix
url, err := url.Parse(s[3:])
if err != nil {
return nil, err
}
if url.Path == "" {
return nil, errors.New("gs: bucket name not found")
}
path := strings.SplitN(url.Path[1:], "/", 2)
// create an s3 configuration using the local
// compatible repository layout
return s3.NewConfig(url.Host, path, url.Scheme == "http", true)
case strings.HasPrefix(s, "gs://"):
s = s[5:]
case strings.HasPrefix(s, "gs:"):
@ -23,6 +44,8 @@ func ParseConfig(s string) (interface{}, error) {
default:
return nil, errors.New(`gcs: config does not start with "gs"`)
}
// use the first entry of the path as bucket and the
// remainder as prefix
p := strings.SplitN(s, "/", 2)
return s3.NewConfig(gcsEndpoint, p, false)
return s3.NewConfig(gcsEndpoint, p, false, true)
}

96
src/restic/backend/gcs/config_test.go
View file

@ -10,44 +10,92 @@ var configTests = []struct {
cfg s3.Config
}{
{"gs://bucketname", s3.Config{
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "restic",
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "restic",
LocalLayout: true,
}},
{"gs://bucketname/", s3.Config{
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "restic",
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "restic",
LocalLayout: true,
}},
{"gs://bucketname/prefix/dir", s3.Config{
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "prefix/dir",
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "prefix/dir",
LocalLayout: true,
}},
{"gs://bucketname/prefix/dir/", s3.Config{
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "prefix/dir",
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "prefix/dir",
LocalLayout: true,
}},
{"gs:bucketname", s3.Config{
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "restic",
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "restic",
LocalLayout: true,
}},
{"gs:bucketname/", s3.Config{
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "restic",
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "restic",
LocalLayout: true,
}},
{"gs:bucketname/prefix/dir", s3.Config{
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "prefix/dir",
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "prefix/dir",
LocalLayout: true,
}},
{"gs:bucketname/prefix/dir/", s3.Config{
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "prefix/dir",
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "prefix/dir",
LocalLayout: true,
}},
{"gs:https://hostname:9999/foobar", s3.Config{
Endpoint: "hostname:9999",
Bucket: "foobar",
Prefix: "restic",
LocalLayout: true,
}},
{"gs:https://hostname:9999/foobar/", s3.Config{
Endpoint: "hostname:9999",
Bucket: "foobar",
Prefix: "restic",
LocalLayout: true,
}},
{"gs:http://hostname:9999/foobar", s3.Config{
Endpoint: "hostname:9999",
Bucket: "foobar",
Prefix: "restic",
UseHTTP: true,
LocalLayout: true,
}},
{"gs:http://hostname:9999/foobar/", s3.Config{
Endpoint: "hostname:9999",
Bucket: "foobar",
Prefix: "restic",
UseHTTP: true,
LocalLayout: true,
}},
{"gs:http://hostname:9999/bucket/prefix/directory", s3.Config{
Endpoint: "hostname:9999",
Bucket: "bucket",
Prefix: "prefix/directory",
UseHTTP: true,
LocalLayout: true,
}},
{"gs:http://hostname:9999/bucket/prefix/directory/", s3.Config{
Endpoint: "hostname:9999",
Bucket: "bucket",
Prefix: "prefix/directory",
UseHTTP: true,
LocalLayout: true,
}},
}

17
src/restic/backend/s3/config.go
View file

@ -16,6 +16,7 @@ type Config struct {
KeyID, Secret string
Bucket string
Prefix string
LocalLayout bool
}
const defaultPrefix = "restic"
@ -25,7 +26,6 @@ const defaultPrefix = "restic"
// s3:host:bucketname/prefix. The host can also be a valid s3 region
// name. If no prefix is given the prefix "restic" will be used.
func ParseConfig(s string) (interface{}, error) {
s = strings.TrimRight(s, "/") // remove trailing slashes
switch {
case strings.HasPrefix(s, "s3:http"):
// assume that a URL has been specified, parse it and
@ -41,7 +41,7 @@ func ParseConfig(s string) (interface{}, error) {
}
path := strings.SplitN(url.Path[1:], "/", 2)
return NewConfig(url.Host, path, url.Scheme == "http")
return NewConfig(url.Host, path, url.Scheme == "http", false)
case strings.HasPrefix(s, "s3://"):
s = s[5:]
case strings.HasPrefix(s, "s3:"):
@ -52,13 +52,13 @@ func ParseConfig(s string) (interface{}, error) {
// use the first entry of the path as the endpoint and the
// remainder as bucket name and prefix
path := strings.SplitN(s, "/", 3)
return NewConfig(path[0], path[1:], false)
return NewConfig(path[0], path[1:], false, false)
}
// NewConfig creates a Config at the specified endpoint. The Bucket is
// the first entry in p and the remaining entries will be used as the
// object name prefix.
func NewConfig(endpoint string, bucketPrefix []string, useHTTP bool) (interface{}, error) {
func NewConfig(endpoint string, bucketPrefix []string, useHTTP bool, localLayout bool) (interface{}, error) {
var prefix string
switch {
case len(bucketPrefix) < 1:
@ -69,9 +69,10 @@ func NewConfig(endpoint string, bucketPrefix []string, useHTTP bool) (interface{
prefix = path.Clean(bucketPrefix[1])
}
return Config{
Endpoint: endpoint,
UseHTTP: useHTTP,
Bucket: bucketPrefix[0],
Prefix: prefix,
Endpoint: endpoint,
UseHTTP: useHTTP,
Bucket: bucketPrefix[0],
Prefix: prefix,
LocalLayout: localLayout,
}, nil
}

48
src/restic/backend/s3/s3.go
View file

@ -9,19 +9,21 @@ import (
"restic/errors"
"github.com/minio/minio-go"
"restic/backend"
"restic/debug"
"github.com/minio/minio-go"
)
const connLimit = 10
// s3 is a backend which stores the data on an S3 endpoint.
type s3 struct {
client *minio.Client
connChan chan struct{}
bucketname string
prefix string
client *minio.Client
connChan chan struct{}
bucketname string
prefix string
localLayout bool
}
// Open opens the S3 backend at bucket and region. The bucket is created if it
@ -34,7 +36,7 @@ func Open(cfg Config) (restic.Backend, error) {
return nil, errors.Wrap(err, "minio.New")
}
be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix}
be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix, localLayout: cfg.LocalLayout}
be.createConnections()
found, err := client.BucketExists(cfg.Bucket)
@ -58,9 +60,41 @@ func (be *s3) s3path(t restic.FileType, name string) string {
if t == restic.ConfigFile {
return path.Join(be.prefix, string(t))
}
if be.localLayout == true {
// use the local compatible repository layout
return path.Join(be.prefix, dirname(t, name), name)
}
// use the standard s3 repository layout
return path.Join(be.prefix, string(t), name)
}
// Construct the directory name for a given FileType and file name using
// the path names defined in backend.Paths.
//
// This will also use the first two characters of the blob name as a
// directory prefix to partition blobs into smaller directories. This
// is similar to the way local and sftp repositories are handling
// blob filenames.
func dirname(t restic.FileType, name string) string {
var n string
switch t {
case restic.DataFile:
n = backend.Paths.Data
if len(name) > 2 {
n = path.Join(n, name[:2])
}
case restic.SnapshotFile:
n = backend.Paths.Snapshots
case restic.IndexFile:
n = backend.Paths.Index
case restic.LockFile:
n = backend.Paths.Locks
case restic.KeyFile:
n = backend.Paths.Keys
}
return n
}
func (be *s3) createConnections() {
be.connChan = make(chan struct{}, connLimit)
for i := 0; i < connLimit; i++ {

28
src/restic/location/location_test.go
View file

@ -56,30 +56,34 @@ var parseTests = []struct {
}}},
{"gs://bucketname", Location{Scheme: "gs",
Config: s3.Config{
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "restic",
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "restic",
LocalLayout: true,
}},
},
{"gs://bucketname/prefix", Location{Scheme: "gs",
Config: s3.Config{
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "prefix",
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "prefix",
LocalLayout: true,
}},
},
{"gs:bucketname", Location{Scheme: "gs",
Config: s3.Config{
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "restic",
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "restic",
LocalLayout: true,
}},
},
{"gs:bucketname/prefix", Location{Scheme: "gs",
Config: s3.Config{
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "prefix",
Endpoint: "storage.googleapis.com",
Bucket: "bucketname",
Prefix: "prefix",
LocalLayout: true,
}},
},
{"s3://eu-central-1/bucketname", Location{Scheme: "s3",