mirror of
https://github.com/restic/restic.git
synced 2025-03-16 00:00:05 +01:00
Added support for a configurable directory prefix for blob names.
Extracted a Filename and Dirname function and used it in the s3 and sftp backends. Added support for setting the length of the directory prefix in the s3 configuration. Introduced a NewConfig method for creating the s3 configuration.
This commit is contained in:
parent
8e3b81c5ec
commit
03209b95fd
5 changed files with 138 additions and 43 deletions
|
@ -1,6 +1,10 @@
|
|||
package backend
|
||||
|
||||
import "os"
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"restic"
|
||||
)
|
||||
|
||||
// Paths contains the default paths for file-based backends (e.g. local).
|
||||
var Paths = struct {
|
||||
|
@ -24,3 +28,53 @@ var Paths = struct {
|
|||
// Modes holds the default modes for directories and files for file-based
|
||||
// backends.
|
||||
var Modes = struct{ Dir, File os.FileMode }{0700, 0600}
|
||||
|
||||
// Join joins the given paths and cleans them afterwards. This always uses
|
||||
// forward slashes, which is required by all non local backends.
|
||||
func Join(parts ...string) string {
|
||||
return path.Clean(path.Join(parts...))
|
||||
}
|
||||
|
||||
// Filename constructs the path for given base name restic.Type and
|
||||
// name.
|
||||
//
|
||||
// This can optionally use a number of characters of the blob name as
|
||||
// a directory prefix to partition blobs into smaller directories.
|
||||
//
|
||||
// Currently local and sftp repositories are handling blob filenames
|
||||
// using a dirPrefixLen of 2.
|
||||
func Filename(base string, t restic.FileType, name string, dirPrefixLen int) string {
|
||||
if t == restic.ConfigFile {
|
||||
return Join(base, Paths.Config)
|
||||
}
|
||||
|
||||
return Join(base, Dirname(t, name, dirPrefixLen), name)
|
||||
}
|
||||
|
||||
// Dirname constructs the directory name for a given FileType and file
|
||||
// name using the path names defined in Paths.
|
||||
//
|
||||
// This can optionally use a number of characters of the blob name as
|
||||
// a directory prefix to partition blobs into smaller directories.
|
||||
//
|
||||
// Currently local and sftp repositories are handling blob filenames
|
||||
// using a dirPrefixLen of 2.
|
||||
func Dirname(t restic.FileType, name string, dirPrefixLen int) string {
|
||||
var n string
|
||||
switch t {
|
||||
case restic.DataFile:
|
||||
n = Paths.Data
|
||||
if dirPrefixLen > 0 && len(name) > dirPrefixLen {
|
||||
n = path.Join(n, name[:dirPrefixLen])
|
||||
}
|
||||
case restic.SnapshotFile:
|
||||
n = Paths.Snapshots
|
||||
case restic.IndexFile:
|
||||
n = Paths.Index
|
||||
case restic.LockFile:
|
||||
n = Paths.Locks
|
||||
case restic.KeyFile:
|
||||
n = Paths.Keys
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
|
55
src/restic/backend/paths_test.go
Normal file
55
src/restic/backend/paths_test.go
Normal file
|
@ -0,0 +1,55 @@
|
|||
package backend_test
|
||||
|
||||
import (
|
||||
"restic"
|
||||
"testing"
|
||||
|
||||
"restic/backend"
|
||||
)
|
||||
|
||||
var pathsTests = []struct {
|
||||
base string
|
||||
t restic.FileType
|
||||
name string
|
||||
prefixLen int
|
||||
res string
|
||||
}{
|
||||
{"base", restic.DataFile, "abcdef", 0, "base/data/abcdef"},
|
||||
{"base", restic.DataFile, "abcdef", 2, "base/data/ab/abcdef"},
|
||||
{"base", restic.DataFile, "abcdef", 4, "base/data/abcd/abcdef"},
|
||||
{"base", restic.DataFile, "ab", 2, "base/data/ab"},
|
||||
{"base", restic.DataFile, "abcd", 4, "base/data/abcd"},
|
||||
|
||||
{"", restic.DataFile, "abcdef", 0, "data/abcdef"},
|
||||
{"", restic.DataFile, "abcdef", 2, "data/ab/abcdef"},
|
||||
{"", restic.DataFile, "abcdef", 4, "data/abcd/abcdef"},
|
||||
{"", restic.DataFile, "ab", 2, "data/ab"},
|
||||
{"", restic.DataFile, "abcd", 4, "data/abcd"},
|
||||
|
||||
{"base", restic.ConfigFile, "file", 0, "base/config"},
|
||||
{"", restic.ConfigFile, "file", 0, "config"},
|
||||
|
||||
{"base", restic.SnapshotFile, "file", 0, "base/snapshots/file"},
|
||||
{"base", restic.SnapshotFile, "file", 2, "base/snapshots/file"},
|
||||
|
||||
{"base", restic.IndexFile, "file", 0, "base/index/file"},
|
||||
{"base", restic.IndexFile, "file", 2, "base/index/file"},
|
||||
|
||||
{"base", restic.LockFile, "file", 0, "base/locks/file"},
|
||||
{"base", restic.LockFile, "file", 2, "base/locks/file"},
|
||||
|
||||
{"base", restic.KeyFile, "file", 0, "base/keys/file"},
|
||||
{"base", restic.KeyFile, "file", 2, "base/keys/file"},
|
||||
}
|
||||
|
||||
func TestFilename(t *testing.T) {
|
||||
for i, test := range pathsTests {
|
||||
res := backend.Filename(test.base, test.t, test.name, test.prefixLen)
|
||||
|
||||
if test.res != res {
|
||||
t.Errorf("test %d: result does not match, want %q, got %q",
|
||||
i, test.res, res)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
|
@ -16,9 +16,13 @@ type Config struct {
|
|||
KeyID, Secret string
|
||||
Bucket string
|
||||
Prefix string
|
||||
DirPrefixLen int
|
||||
}
|
||||
|
||||
const defaultPrefix = "restic"
|
||||
const (
|
||||
defaultPrefix = "restic"
|
||||
defaultDirPrefixLen = 0
|
||||
)
|
||||
|
||||
// ParseConfig parses the string s and extracts the s3 config. The two
|
||||
// supported configuration formats are s3://host/bucketname/prefix and
|
||||
|
@ -40,7 +44,7 @@ func ParseConfig(s string) (interface{}, error) {
|
|||
}
|
||||
|
||||
path := strings.SplitN(url.Path[1:], "/", 2)
|
||||
return createConfig(url.Host, path, url.Scheme == "http")
|
||||
return NewConfig(url.Host, path, url.Scheme == "http", defaultDirPrefixLen)
|
||||
case strings.HasPrefix(s, "s3://"):
|
||||
s = s[5:]
|
||||
case strings.HasPrefix(s, "s3:"):
|
||||
|
@ -51,10 +55,13 @@ func ParseConfig(s string) (interface{}, error) {
|
|||
// use the first entry of the path as the endpoint and the
|
||||
// remainder as bucket name and prefix
|
||||
path := strings.SplitN(s, "/", 3)
|
||||
return createConfig(path[0], path[1:], false)
|
||||
return NewConfig(path[0], path[1:], false, defaultDirPrefixLen)
|
||||
}
|
||||
|
||||
func createConfig(endpoint string, p []string, useHTTP bool) (interface{}, error) {
|
||||
// NewConfig creates a Config at the specified endpoint. The Bucket is
|
||||
// the first entry in path and the remaining entries will be used as the
|
||||
// object name prefix.
|
||||
func NewConfig(endpoint string, p []string, useHTTP bool, dirPrefixLen int) (interface{}, error) {
|
||||
var prefix string
|
||||
switch {
|
||||
case len(p) < 1:
|
||||
|
@ -65,9 +72,10 @@ func createConfig(endpoint string, p []string, useHTTP bool) (interface{}, error
|
|||
prefix = path.Clean(p[1])
|
||||
}
|
||||
return Config{
|
||||
Endpoint: endpoint,
|
||||
UseHTTP: useHTTP,
|
||||
Bucket: p[0],
|
||||
Prefix: prefix,
|
||||
Endpoint: endpoint,
|
||||
UseHTTP: useHTTP,
|
||||
Bucket: p[0],
|
||||
Prefix: prefix,
|
||||
DirPrefixLen: dirPrefixLen,
|
||||
}, nil
|
||||
}
|
||||
|
|
|
@ -3,25 +3,25 @@ package s3
|
|||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"path"
|
||||
"restic"
|
||||
"strings"
|
||||
|
||||
"restic/backend"
|
||||
"restic/debug"
|
||||
"restic/errors"
|
||||
|
||||
"github.com/minio/minio-go"
|
||||
|
||||
"restic/debug"
|
||||
)
|
||||
|
||||
const connLimit = 10
|
||||
|
||||
// s3 is a backend which stores the data on an S3 endpoint.
|
||||
type s3 struct {
|
||||
client *minio.Client
|
||||
connChan chan struct{}
|
||||
bucketname string
|
||||
prefix string
|
||||
client *minio.Client
|
||||
connChan chan struct{}
|
||||
bucketname string
|
||||
prefix string
|
||||
dirPrefixLen int
|
||||
}
|
||||
|
||||
// Open opens the S3 backend at bucket and region. The bucket is created if it
|
||||
|
@ -34,7 +34,7 @@ func Open(cfg Config) (restic.Backend, error) {
|
|||
return nil, errors.Wrap(err, "minio.New")
|
||||
}
|
||||
|
||||
be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix}
|
||||
be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix, dirPrefixLen: cfg.DirPrefixLen}
|
||||
be.createConnections()
|
||||
|
||||
found, err := client.BucketExists(cfg.Bucket)
|
||||
|
@ -55,10 +55,7 @@ func Open(cfg Config) (restic.Backend, error) {
|
|||
}
|
||||
|
||||
func (be *s3) s3path(t restic.FileType, name string) string {
|
||||
if t == restic.ConfigFile {
|
||||
return path.Join(be.prefix, string(t))
|
||||
}
|
||||
return path.Join(be.prefix, string(t), name)
|
||||
return backend.Filename(be.prefix, t, name, be.dirPrefixLen)
|
||||
}
|
||||
|
||||
func (be *s3) createConnections() {
|
||||
|
|
|
@ -23,6 +23,7 @@ import (
|
|||
|
||||
const (
|
||||
tempfileRandomSuffixLength = 10
|
||||
defaultDirPrefixLen = 2
|
||||
)
|
||||
|
||||
// SFTP is a backend in a directory accessed via SFTP.
|
||||
|
@ -298,32 +299,12 @@ func Join(parts ...string) string {
|
|||
|
||||
// Construct path for given restic.Type and name.
|
||||
func (r *SFTP) filename(t restic.FileType, name string) string {
|
||||
if t == restic.ConfigFile {
|
||||
return Join(r.p, "config")
|
||||
}
|
||||
|
||||
return Join(r.dirname(t, name), name)
|
||||
return backend.Filename(r.p, t, name, defaultDirPrefixLen)
|
||||
}
|
||||
|
||||
// Construct directory for given backend.Type.
|
||||
func (r *SFTP) dirname(t restic.FileType, name string) string {
|
||||
var n string
|
||||
switch t {
|
||||
case restic.DataFile:
|
||||
n = backend.Paths.Data
|
||||
if len(name) > 2 {
|
||||
n = Join(n, name[:2])
|
||||
}
|
||||
case restic.SnapshotFile:
|
||||
n = backend.Paths.Snapshots
|
||||
case restic.IndexFile:
|
||||
n = backend.Paths.Index
|
||||
case restic.LockFile:
|
||||
n = backend.Paths.Locks
|
||||
case restic.KeyFile:
|
||||
n = backend.Paths.Keys
|
||||
}
|
||||
return Join(r.p, n)
|
||||
return Join(r.p, backend.Dirname(t, name, defaultDirPrefixLen))
|
||||
}
|
||||
|
||||
// Load returns the data stored in the backend for h at the given offset
|
||||
|
|
Loading…
Add table
Reference in a new issue