Align path names generated for file and S3 backend

This aligns the path names generated for S3 backend to the ones used by
the file backend allowing S3 objects to be used as file backend and
vice versa.

Dirname and Filename generation logic have moved from file backend to
tha backend package.

Added a environment variable (AWS_LEGACY_PATHS) to S3 backend which
cat be set to true to switch to legacy pathnames (to be used with
existing repositories).

Fixes #654

src/cmds/restic/global.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"restic"
 	"runtime"
+	"strconv"
 	"strings"
 	"syscall"
 
@@ -326,6 +327,7 @@ func open(s string) (restic.Backend, error) {
 		if cfg.Secret == "" {
 			cfg.Secret = os.Getenv("AWS_SECRET_ACCESS_KEY")
 		}
+		cfg.LegacyPaths, _ = strconv.ParseBool(os.Getenv("AWS_LEGACY_PATHS"))
 
 		debug.Log("opening s3 repository at %#v", cfg)
 		be, err = s3.Open(cfg)

src/restic/backend/local/local.go

@@ -71,36 +71,6 @@ func (b *Local) Location() string {
 	return b.p
 }
 
-// Construct path for given Type and name.
-func filename(base string, t restic.FileType, name string) string {
-	if t == restic.ConfigFile {
-		return filepath.Join(base, "config")
-	}
-
-	return filepath.Join(dirname(base, t, name), name)
-}
-
-// Construct directory for given Type.
-func dirname(base string, t restic.FileType, name string) string {
-	var n string
-	switch t {
-	case restic.DataFile:
-		n = backend.Paths.Data
-		if len(name) > 2 {
-			n = filepath.Join(n, name[:2])
-		}
-	case restic.SnapshotFile:
-		n = backend.Paths.Snapshots
-	case restic.IndexFile:
-		n = backend.Paths.Index
-	case restic.LockFile:
-		n = backend.Paths.Locks
-	case restic.KeyFile:
-		n = backend.Paths.Keys
-	}
-	return filepath.Join(base, n)
-}
-
 // Load returns the data stored in the backend for h at the given offset and
 // saves it in p. Load has the same semantics as io.ReaderAt, with one
 // exception: when off is lower than zero, it is treated as an offset relative
@@ -111,7 +81,7 @@ func (b *Local) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
 		return 0, err
 	}
 
-	f, err := fs.Open(filename(b.p, h.Type, h.Name))
+	f, err := fs.Open(backend.Filename(b.p, h.Type, h.Name))
 	if err != nil {
 		return 0, errors.Wrap(err, "Open")
 	}
@@ -178,7 +148,7 @@ func (b *Local) Save(h restic.Handle, p []byte) (err error) {
 		return err
 	}
 
-	filename := filename(b.p, h.Type, h.Name)
+	filename := backend.Filename(b.p, h.Type, h.Name)
 
 	// test if new path already exists
 	if _, err := fs.Stat(filename); err == nil {
@@ -217,7 +187,7 @@ func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) {
 		return restic.FileInfo{}, err
 	}
 
-	fi, err := fs.Stat(filename(b.p, h.Type, h.Name))
+	fi, err := fs.Stat(backend.Filename(b.p, h.Type, h.Name))
 	if err != nil {
 		return restic.FileInfo{}, errors.Wrap(err, "Stat")
 	}
@@ -228,7 +198,7 @@ func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) {
 // Test returns true if a blob of the given type and name exists in the backend.
 func (b *Local) Test(t restic.FileType, name string) (bool, error) {
 	debug.Log("Test %v %v", t, name)
-	_, err := fs.Stat(filename(b.p, t, name))
+	_, err := fs.Stat(backend.Filename(b.p, t, name))
 	if err != nil {
 		if os.IsNotExist(errors.Cause(err)) {
 			return false, nil
@@ -242,7 +212,7 @@ func (b *Local) Test(t restic.FileType, name string) (bool, error) {
 // Remove removes the blob with the given name and type.
 func (b *Local) Remove(t restic.FileType, name string) error {
 	debug.Log("Remove %v %v", t, name)
-	fn := filename(b.p, t, name)
+	fn := backend.Filename(b.p, t, name)
 
 	// reset read-only flag
 	err := fs.Chmod(fn, 0666)
@@ -323,7 +293,7 @@ func (b *Local) List(t restic.FileType, done <-chan struct{}) <-chan string {
 	}
 
 	ch := make(chan string)
-	items, err := lister(filepath.Join(dirname(b.p, t, "")))
+	items, err := lister(filepath.Join(backend.Dirname(b.p, t, "")))
 	if err != nil {
 		close(ch)
 		return ch

src/restic/backend/paths.go

@@ -1,6 +1,11 @@
 package backend
 
-import "os"
+import (
+	"os"
+	"path/filepath"
+
+	"restic"
+)
 
 // Paths contains the default paths for file-based backends (e.g. local).
 var Paths = struct {
@@ -24,3 +29,33 @@ var Paths = struct {
 // Modes holds the default modes for directories and files for file-based
 // backends.
 var Modes = struct{ Dir, File os.FileMode }{0700, 0600}
+
+// Construct default directory for given FileType.
+func Dirname(base string, t restic.FileType, name string) string {
+	var n string
+	switch t {
+	case restic.DataFile:
+		n = Paths.Data
+		if len(name) > 2 {
+			n = filepath.Join(n, name[:2])
+		}
+	case restic.SnapshotFile:
+		n = Paths.Snapshots
+	case restic.IndexFile:
+		n = Paths.Index
+	case restic.LockFile:
+		n = Paths.Locks
+	case restic.KeyFile:
+		n = Paths.Keys
+	}
+	return filepath.Join(base, n)
+}
+
+// Construct default path for given FileType and name.
+func Filename(base string, t restic.FileType, name string) string {
+	if t == restic.ConfigFile {
+		return filepath.Join(base, "config")
+	}
+
+	return filepath.Join(Dirname(base, t, name), name)
+}

src/restic/backend/s3/config.go

@@ -13,6 +13,7 @@ import (
 type Config struct {
 	Endpoint      string
 	UseHTTP       bool
+	LegacyPaths   bool
 	KeyID, Secret string
 	Bucket        string
 	Prefix        string

src/restic/backend/s3/s3.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/minio/minio-go"
 
+	"restic/backend"
 	"restic/debug"
 )
 
@@ -18,10 +19,11 @@ const connLimit = 10
 
 // s3 is a backend which stores the data on an S3 endpoint.
 type s3 struct {
-	client     *minio.Client
-	connChan   chan struct{}
-	bucketname string
-	prefix     string
+	client      *minio.Client
+	connChan    chan struct{}
+	bucketname  string
+	prefix      string
+	legacyPaths bool
 }
 
 // Open opens the S3 backend at bucket and region. The bucket is created if it
@@ -34,7 +36,7 @@ func Open(cfg Config) (restic.Backend, error) {
 		return nil, errors.Wrap(err, "minio.New")
 	}
 
-	be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix}
+	be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix, legacyPaths: cfg.LegacyPaths}
 	be.createConnections()
 
 	found, err := client.BucketExists(cfg.Bucket)
@@ -55,10 +57,14 @@ func Open(cfg Config) (restic.Backend, error) {
 }
 
 func (be *s3) s3path(t restic.FileType, name string) string {
-	if t == restic.ConfigFile {
-		return path.Join(be.prefix, string(t))
+	// Use legacy paths if requested
+	if be.legacyPaths {
+		if t == restic.ConfigFile {
+			return path.Join(be.prefix, string(t))
+		}
+		return path.Join(be.prefix, string(t), name)
 	}
-	return path.Join(be.prefix, string(t), name)
+	return backend.Filename(be.prefix, t, name)
 }
 
 func (be *s3) createConnections() {