1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2025-03-16 00:00:05 +01:00

Added support for a configurable directory prefix for blob names.

Extracted a Filename and Dirname function and used it in the s3 and
sftp backends. Added support for setting the length of the directory
prefix in the s3 configuration. Introduced a NewConfig method for
creating the s3 configuration.
This commit is contained in:
Christian Kemper 2016-12-30 14:59:34 -08:00
parent 8e3b81c5ec
commit 03209b95fd
5 changed files with 138 additions and 43 deletions

View file

@ -1,6 +1,10 @@
package backend
import "os"
import (
"os"
"path"
"restic"
)
// Paths contains the default paths for file-based backends (e.g. local).
var Paths = struct {
@ -24,3 +28,53 @@ var Paths = struct {
// Modes holds the default modes for directories and files for file-based
// backends.
var Modes = struct{ Dir, File os.FileMode }{0700, 0600}
// Join joins the given paths and cleans them afterwards. This always uses
// forward slashes, which is required by all non local backends.
func Join(parts ...string) string {
return path.Clean(path.Join(parts...))
}
// Filename constructs the path for given base name restic.Type and
// name.
//
// This can optionally use a number of characters of the blob name as
// a directory prefix to partition blobs into smaller directories.
//
// Currently local and sftp repositories are handling blob filenames
// using a dirPrefixLen of 2.
func Filename(base string, t restic.FileType, name string, dirPrefixLen int) string {
if t == restic.ConfigFile {
return Join(base, Paths.Config)
}
return Join(base, Dirname(t, name, dirPrefixLen), name)
}
// Dirname constructs the directory name for a given FileType and file
// name using the path names defined in Paths.
//
// This can optionally use a number of characters of the blob name as
// a directory prefix to partition blobs into smaller directories.
//
// Currently local and sftp repositories are handling blob filenames
// using a dirPrefixLen of 2.
func Dirname(t restic.FileType, name string, dirPrefixLen int) string {
var n string
switch t {
case restic.DataFile:
n = Paths.Data
if dirPrefixLen > 0 && len(name) > dirPrefixLen {
n = path.Join(n, name[:dirPrefixLen])
}
case restic.SnapshotFile:
n = Paths.Snapshots
case restic.IndexFile:
n = Paths.Index
case restic.LockFile:
n = Paths.Locks
case restic.KeyFile:
n = Paths.Keys
}
return n
}

View file

@ -0,0 +1,55 @@
package backend_test
import (
"restic"
"testing"
"restic/backend"
)
var pathsTests = []struct {
base string
t restic.FileType
name string
prefixLen int
res string
}{
{"base", restic.DataFile, "abcdef", 0, "base/data/abcdef"},
{"base", restic.DataFile, "abcdef", 2, "base/data/ab/abcdef"},
{"base", restic.DataFile, "abcdef", 4, "base/data/abcd/abcdef"},
{"base", restic.DataFile, "ab", 2, "base/data/ab"},
{"base", restic.DataFile, "abcd", 4, "base/data/abcd"},
{"", restic.DataFile, "abcdef", 0, "data/abcdef"},
{"", restic.DataFile, "abcdef", 2, "data/ab/abcdef"},
{"", restic.DataFile, "abcdef", 4, "data/abcd/abcdef"},
{"", restic.DataFile, "ab", 2, "data/ab"},
{"", restic.DataFile, "abcd", 4, "data/abcd"},
{"base", restic.ConfigFile, "file", 0, "base/config"},
{"", restic.ConfigFile, "file", 0, "config"},
{"base", restic.SnapshotFile, "file", 0, "base/snapshots/file"},
{"base", restic.SnapshotFile, "file", 2, "base/snapshots/file"},
{"base", restic.IndexFile, "file", 0, "base/index/file"},
{"base", restic.IndexFile, "file", 2, "base/index/file"},
{"base", restic.LockFile, "file", 0, "base/locks/file"},
{"base", restic.LockFile, "file", 2, "base/locks/file"},
{"base", restic.KeyFile, "file", 0, "base/keys/file"},
{"base", restic.KeyFile, "file", 2, "base/keys/file"},
}
func TestFilename(t *testing.T) {
for i, test := range pathsTests {
res := backend.Filename(test.base, test.t, test.name, test.prefixLen)
if test.res != res {
t.Errorf("test %d: result does not match, want %q, got %q",
i, test.res, res)
}
}
}

View file

@ -16,9 +16,13 @@ type Config struct {
KeyID, Secret string
Bucket string
Prefix string
DirPrefixLen int
}
const defaultPrefix = "restic"
const (
defaultPrefix = "restic"
defaultDirPrefixLen = 0
)
// ParseConfig parses the string s and extracts the s3 config. The two
// supported configuration formats are s3://host/bucketname/prefix and
@ -40,7 +44,7 @@ func ParseConfig(s string) (interface{}, error) {
}
path := strings.SplitN(url.Path[1:], "/", 2)
return createConfig(url.Host, path, url.Scheme == "http")
return NewConfig(url.Host, path, url.Scheme == "http", defaultDirPrefixLen)
case strings.HasPrefix(s, "s3://"):
s = s[5:]
case strings.HasPrefix(s, "s3:"):
@ -51,10 +55,13 @@ func ParseConfig(s string) (interface{}, error) {
// use the first entry of the path as the endpoint and the
// remainder as bucket name and prefix
path := strings.SplitN(s, "/", 3)
return createConfig(path[0], path[1:], false)
return NewConfig(path[0], path[1:], false, defaultDirPrefixLen)
}
func createConfig(endpoint string, p []string, useHTTP bool) (interface{}, error) {
// NewConfig creates a Config at the specified endpoint. The Bucket is
// the first entry in path and the remaining entries will be used as the
// object name prefix.
func NewConfig(endpoint string, p []string, useHTTP bool, dirPrefixLen int) (interface{}, error) {
var prefix string
switch {
case len(p) < 1:
@ -65,9 +72,10 @@ func createConfig(endpoint string, p []string, useHTTP bool) (interface{}, error
prefix = path.Clean(p[1])
}
return Config{
Endpoint: endpoint,
UseHTTP: useHTTP,
Bucket: p[0],
Prefix: prefix,
Endpoint: endpoint,
UseHTTP: useHTTP,
Bucket: p[0],
Prefix: prefix,
DirPrefixLen: dirPrefixLen,
}, nil
}

View file

@ -3,25 +3,25 @@ package s3
import (
"bytes"
"io"
"path"
"restic"
"strings"
"restic/backend"
"restic/debug"
"restic/errors"
"github.com/minio/minio-go"
"restic/debug"
)
const connLimit = 10
// s3 is a backend which stores the data on an S3 endpoint.
type s3 struct {
client *minio.Client
connChan chan struct{}
bucketname string
prefix string
client *minio.Client
connChan chan struct{}
bucketname string
prefix string
dirPrefixLen int
}
// Open opens the S3 backend at bucket and region. The bucket is created if it
@ -34,7 +34,7 @@ func Open(cfg Config) (restic.Backend, error) {
return nil, errors.Wrap(err, "minio.New")
}
be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix}
be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix, dirPrefixLen: cfg.DirPrefixLen}
be.createConnections()
found, err := client.BucketExists(cfg.Bucket)
@ -55,10 +55,7 @@ func Open(cfg Config) (restic.Backend, error) {
}
func (be *s3) s3path(t restic.FileType, name string) string {
if t == restic.ConfigFile {
return path.Join(be.prefix, string(t))
}
return path.Join(be.prefix, string(t), name)
return backend.Filename(be.prefix, t, name, be.dirPrefixLen)
}
func (be *s3) createConnections() {

View file

@ -23,6 +23,7 @@ import (
const (
tempfileRandomSuffixLength = 10
defaultDirPrefixLen = 2
)
// SFTP is a backend in a directory accessed via SFTP.
@ -298,32 +299,12 @@ func Join(parts ...string) string {
// Construct path for given restic.Type and name.
func (r *SFTP) filename(t restic.FileType, name string) string {
if t == restic.ConfigFile {
return Join(r.p, "config")
}
return Join(r.dirname(t, name), name)
return backend.Filename(r.p, t, name, defaultDirPrefixLen)
}
// Construct directory for given backend.Type.
func (r *SFTP) dirname(t restic.FileType, name string) string {
var n string
switch t {
case restic.DataFile:
n = backend.Paths.Data
if len(name) > 2 {
n = Join(n, name[:2])
}
case restic.SnapshotFile:
n = backend.Paths.Snapshots
case restic.IndexFile:
n = backend.Paths.Index
case restic.LockFile:
n = backend.Paths.Locks
case restic.KeyFile:
n = backend.Paths.Keys
}
return Join(r.p, n)
return Join(r.p, backend.Dirname(t, name, defaultDirPrefixLen))
}
// Load returns the data stored in the backend for h at the given offset