mirror of
https://github.com/restic/restic.git
synced 2025-03-16 00:00:05 +01:00
Merge remote-tracking branch 'old-origin/master' into add-smb-backend
This commit is contained in:
commit
4275870876
50 changed files with 966 additions and 758 deletions
11
changelog/unreleased/issue-3397
Normal file
11
changelog/unreleased/issue-3397
Normal file
|
@ -0,0 +1,11 @@
|
|||
Enhancement: Improve the ETA displayed during backup
|
||||
|
||||
Restic's `backup` command displayed an ETA that did not adapt when the rate
|
||||
of progress made during the backup changed during the course of the
|
||||
backup. Restic now uses recent progress when computing the ETA. It is
|
||||
important to realize that the estimate may still be wrong, because restic
|
||||
cannot predict the future, but the hope is that the ETA will be more
|
||||
accurate in most cases.
|
||||
|
||||
https://github.com/restic/restic/issues/3397
|
||||
https://github.com/restic/restic/pull/3563
|
8
changelog/unreleased/pull-3261
Normal file
8
changelog/unreleased/pull-3261
Normal file
|
@ -0,0 +1,8 @@
|
|||
Enhancement: Reduce file fragmentation for local backend
|
||||
|
||||
Before this change, local backend files could become fragmented.
|
||||
Now restic will try to preallocate space for pack files to avoid
|
||||
their fragmentation.
|
||||
|
||||
https://github.com/restic/restic/issues/2679
|
||||
https://github.com/restic/restic/pull/3261
|
|
@ -536,170 +536,21 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi
|
|||
}
|
||||
|
||||
func parseConfig(loc location.Location, opts options.Options) (interface{}, error) {
|
||||
cfg := loc.Config
|
||||
if cfg, ok := cfg.(restic.ApplyEnvironmenter); ok {
|
||||
if err := cfg.ApplyEnvironment(""); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// only apply options for a particular backend here
|
||||
opts = opts.Extract(loc.Scheme)
|
||||
|
||||
switch loc.Scheme {
|
||||
case "local":
|
||||
cfg := loc.Config.(local.Config)
|
||||
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
|
||||
if err := opts.Apply(loc.Scheme, cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
debug.Log("opening local repository at %#v", cfg)
|
||||
debug.Log("opening %v repository at %#v", loc.Scheme, cfg)
|
||||
return cfg, nil
|
||||
|
||||
case "sftp":
|
||||
cfg := loc.Config.(sftp.Config)
|
||||
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
debug.Log("opening sftp repository at %#v", cfg)
|
||||
return cfg, nil
|
||||
|
||||
case "s3":
|
||||
cfg := loc.Config.(s3.Config)
|
||||
if cfg.KeyID == "" {
|
||||
cfg.KeyID = os.Getenv("AWS_ACCESS_KEY_ID")
|
||||
}
|
||||
|
||||
if cfg.Secret.String() == "" {
|
||||
cfg.Secret = options.NewSecretString(os.Getenv("AWS_SECRET_ACCESS_KEY"))
|
||||
}
|
||||
|
||||
if cfg.KeyID == "" && cfg.Secret.String() != "" {
|
||||
return nil, errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty")
|
||||
} else if cfg.KeyID != "" && cfg.Secret.String() == "" {
|
||||
return nil, errors.Fatalf("unable to open S3 backend: Secret ($AWS_SECRET_ACCESS_KEY) is empty")
|
||||
}
|
||||
|
||||
if cfg.Region == "" {
|
||||
cfg.Region = os.Getenv("AWS_DEFAULT_REGION")
|
||||
}
|
||||
|
||||
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
debug.Log("opening s3 repository at %#v", cfg)
|
||||
return cfg, nil
|
||||
|
||||
case "gs":
|
||||
cfg := loc.Config.(gs.Config)
|
||||
if cfg.ProjectID == "" {
|
||||
cfg.ProjectID = os.Getenv("GOOGLE_PROJECT_ID")
|
||||
}
|
||||
|
||||
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
debug.Log("opening gs repository at %#v", cfg)
|
||||
return cfg, nil
|
||||
|
||||
case "azure":
|
||||
cfg := loc.Config.(azure.Config)
|
||||
if cfg.AccountName == "" {
|
||||
cfg.AccountName = os.Getenv("AZURE_ACCOUNT_NAME")
|
||||
}
|
||||
|
||||
if cfg.AccountKey.String() == "" {
|
||||
cfg.AccountKey = options.NewSecretString(os.Getenv("AZURE_ACCOUNT_KEY"))
|
||||
}
|
||||
|
||||
if cfg.AccountSAS.String() == "" {
|
||||
cfg.AccountSAS = options.NewSecretString(os.Getenv("AZURE_ACCOUNT_SAS"))
|
||||
}
|
||||
|
||||
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
debug.Log("opening gs repository at %#v", cfg)
|
||||
return cfg, nil
|
||||
|
||||
case "swift":
|
||||
cfg := loc.Config.(swift.Config)
|
||||
|
||||
if err := swift.ApplyEnvironment("", &cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
debug.Log("opening swift repository at %#v", cfg)
|
||||
return cfg, nil
|
||||
|
||||
case "b2":
|
||||
cfg := loc.Config.(b2.Config)
|
||||
|
||||
if cfg.AccountID == "" {
|
||||
cfg.AccountID = os.Getenv("B2_ACCOUNT_ID")
|
||||
}
|
||||
|
||||
if cfg.AccountID == "" {
|
||||
return nil, errors.Fatalf("unable to open B2 backend: Account ID ($B2_ACCOUNT_ID) is empty")
|
||||
}
|
||||
|
||||
if cfg.Key.String() == "" {
|
||||
cfg.Key = options.NewSecretString(os.Getenv("B2_ACCOUNT_KEY"))
|
||||
}
|
||||
|
||||
if cfg.Key.String() == "" {
|
||||
return nil, errors.Fatalf("unable to open B2 backend: Key ($B2_ACCOUNT_KEY) is empty")
|
||||
}
|
||||
|
||||
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
debug.Log("opening b2 repository at %#v", cfg)
|
||||
return cfg, nil
|
||||
case "rest":
|
||||
cfg := loc.Config.(rest.Config)
|
||||
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
debug.Log("opening rest repository at %#v", cfg)
|
||||
return cfg, nil
|
||||
case "rclone":
|
||||
cfg := loc.Config.(rclone.Config)
|
||||
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
debug.Log("opening rest repository at %#v", cfg)
|
||||
return cfg, nil
|
||||
case "smb":
|
||||
cfg := loc.Config.(smb.Config)
|
||||
if cfg.User == "" {
|
||||
cfg.User = os.Getenv("RESTIC_SMB_USER")
|
||||
}
|
||||
|
||||
if cfg.Password.String() == "" {
|
||||
cfg.Password = options.NewSecretString(os.Getenv("RESTIC_SMB_PASSWORD"))
|
||||
}
|
||||
|
||||
if cfg.Domain == "" {
|
||||
cfg.Domain = os.Getenv("RESTIC_SMB_DOMAIN")
|
||||
}
|
||||
if cfg.Domain == "" {
|
||||
cfg.Domain = smb.DefaultDomain
|
||||
}
|
||||
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
debug.Log("opening smb repository at %#v", cfg)
|
||||
return cfg, nil
|
||||
|
||||
}
|
||||
|
||||
return nil, errors.Fatalf("invalid backend: %q", loc.Scheme)
|
||||
}
|
||||
|
||||
// Open the backend specified by a location config.
|
||||
|
@ -728,25 +579,25 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio
|
|||
|
||||
switch loc.Scheme {
|
||||
case "local":
|
||||
be, err = local.Open(ctx, cfg.(local.Config))
|
||||
be, err = local.Open(ctx, *cfg.(*local.Config))
|
||||
case "sftp":
|
||||
be, err = sftp.Open(ctx, cfg.(sftp.Config))
|
||||
case "smb":
|
||||
be, err = smb.Open(ctx, cfg.(smb.Config))
|
||||
be, err = sftp.Open(ctx, *cfg.(*sftp.Config))
|
||||
case "s3":
|
||||
be, err = s3.Open(ctx, cfg.(s3.Config), rt)
|
||||
be, err = s3.Open(ctx, *cfg.(*s3.Config), rt)
|
||||
case "gs":
|
||||
be, err = gs.Open(cfg.(gs.Config), rt)
|
||||
be, err = gs.Open(*cfg.(*gs.Config), rt)
|
||||
case "azure":
|
||||
be, err = azure.Open(ctx, cfg.(azure.Config), rt)
|
||||
be, err = azure.Open(ctx, *cfg.(*azure.Config), rt)
|
||||
case "swift":
|
||||
be, err = swift.Open(ctx, cfg.(swift.Config), rt)
|
||||
be, err = swift.Open(ctx, *cfg.(*swift.Config), rt)
|
||||
case "b2":
|
||||
be, err = b2.Open(ctx, cfg.(b2.Config), rt)
|
||||
be, err = b2.Open(ctx, *cfg.(*b2.Config), rt)
|
||||
case "rest":
|
||||
be, err = rest.Open(cfg.(rest.Config), rt)
|
||||
be, err = rest.Open(*cfg.(*rest.Config), rt)
|
||||
case "rclone":
|
||||
be, err = rclone.Open(cfg.(rclone.Config), lim)
|
||||
be, err = rclone.Open(*cfg.(*rclone.Config), lim)
|
||||
case "smb":
|
||||
be, err = smb.Open(ctx, *cfg.(*smb.Config))
|
||||
|
||||
default:
|
||||
return nil, errors.Fatalf("invalid backend: %q", loc.Scheme)
|
||||
|
@ -806,25 +657,25 @@ func create(ctx context.Context, s string, opts options.Options) (restic.Backend
|
|||
var be restic.Backend
|
||||
switch loc.Scheme {
|
||||
case "local":
|
||||
be, err = local.Create(ctx, cfg.(local.Config))
|
||||
be, err = local.Create(ctx, *cfg.(*local.Config))
|
||||
case "sftp":
|
||||
be, err = sftp.Create(ctx, cfg.(sftp.Config))
|
||||
case "smb":
|
||||
be, err = smb.Create(ctx, cfg.(smb.Config))
|
||||
be, err = sftp.Create(ctx, *cfg.(*sftp.Config))
|
||||
case "s3":
|
||||
be, err = s3.Create(ctx, cfg.(s3.Config), rt)
|
||||
be, err = s3.Create(ctx, *cfg.(*s3.Config), rt)
|
||||
case "gs":
|
||||
be, err = gs.Create(ctx, cfg.(gs.Config), rt)
|
||||
be, err = gs.Create(ctx, *cfg.(*gs.Config), rt)
|
||||
case "azure":
|
||||
be, err = azure.Create(ctx, cfg.(azure.Config), rt)
|
||||
be, err = azure.Create(ctx, *cfg.(*azure.Config), rt)
|
||||
case "swift":
|
||||
be, err = swift.Open(ctx, cfg.(swift.Config), rt)
|
||||
be, err = swift.Open(ctx, *cfg.(*swift.Config), rt)
|
||||
case "b2":
|
||||
be, err = b2.Create(ctx, cfg.(b2.Config), rt)
|
||||
be, err = b2.Create(ctx, *cfg.(*b2.Config), rt)
|
||||
case "rest":
|
||||
be, err = rest.Create(ctx, cfg.(rest.Config), rt)
|
||||
be, err = rest.Create(ctx, *cfg.(*rest.Config), rt)
|
||||
case "rclone":
|
||||
be, err = rclone.Create(ctx, cfg.(rclone.Config))
|
||||
be, err = rclone.Create(ctx, *cfg.(*rclone.Config))
|
||||
case "smb":
|
||||
be, err = smb.Create(ctx, *cfg.(*smb.Config))
|
||||
default:
|
||||
debug.Log("invalid repository scheme: %v", s)
|
||||
return nil, errors.Fatalf("invalid scheme %q", loc.Scheme)
|
||||
|
|
|
@ -18,34 +18,34 @@ import (
|
|||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
func newAzureTestSuite(t testing.TB) *test.Suite {
|
||||
func newAzureTestSuite(t testing.TB) *test.Suite[azure.Config] {
|
||||
tr, err := backend.Transport(backend.TransportOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("cannot create transport for tests: %v", err)
|
||||
}
|
||||
|
||||
return &test.Suite{
|
||||
return &test.Suite[azure.Config]{
|
||||
// do not use excessive data
|
||||
MinimalData: true,
|
||||
|
||||
// NewConfig returns a config for a new temporary backend that will be used in tests.
|
||||
NewConfig: func() (interface{}, error) {
|
||||
azcfg, err := azure.ParseConfig(os.Getenv("RESTIC_TEST_AZURE_REPOSITORY"))
|
||||
NewConfig: func() (*azure.Config, error) {
|
||||
cfg, err := azure.ParseConfig(os.Getenv("RESTIC_TEST_AZURE_REPOSITORY"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = cfg.ApplyEnvironment("RESTIC_TEST_")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg := azcfg.(azure.Config)
|
||||
cfg.AccountName = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_NAME")
|
||||
cfg.AccountKey = options.NewSecretString(os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY"))
|
||||
cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
|
||||
return cfg, nil
|
||||
},
|
||||
|
||||
// CreateFn is a function that creates a temporary repository for the tests.
|
||||
Create: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(azure.Config)
|
||||
|
||||
Create: func(cfg azure.Config) (restic.Backend, error) {
|
||||
ctx := context.TODO()
|
||||
be, err := azure.Create(ctx, cfg, tr)
|
||||
if err != nil {
|
||||
|
@ -65,15 +65,13 @@ func newAzureTestSuite(t testing.TB) *test.Suite {
|
|||
},
|
||||
|
||||
// OpenFn is a function that opens a previously created temporary repository.
|
||||
Open: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(azure.Config)
|
||||
Open: func(cfg azure.Config) (restic.Backend, error) {
|
||||
ctx := context.TODO()
|
||||
return azure.Open(ctx, cfg, tr)
|
||||
},
|
||||
|
||||
// CleanupFn removes data created during the tests.
|
||||
Cleanup: func(config interface{}) error {
|
||||
cfg := config.(azure.Config)
|
||||
Cleanup: func(cfg azure.Config) error {
|
||||
ctx := context.TODO()
|
||||
be, err := azure.Open(ctx, cfg, tr)
|
||||
if err != nil {
|
||||
|
@ -141,12 +139,11 @@ func TestUploadLargeFile(t *testing.T) {
|
|||
return
|
||||
}
|
||||
|
||||
azcfg, err := azure.ParseConfig(os.Getenv("RESTIC_TEST_AZURE_REPOSITORY"))
|
||||
cfg, err := azure.ParseConfig(os.Getenv("RESTIC_TEST_AZURE_REPOSITORY"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cfg := azcfg.(azure.Config)
|
||||
cfg.AccountName = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_NAME")
|
||||
cfg.AccountKey = options.NewSecretString(os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY"))
|
||||
cfg.Prefix = fmt.Sprintf("test-upload-large-%d", time.Now().UnixNano())
|
||||
|
@ -156,7 +153,7 @@ func TestUploadLargeFile(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
|
||||
be, err := azure.Create(ctx, cfg, tr)
|
||||
be, err := azure.Create(ctx, *cfg, tr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
|
@ -1,11 +1,13 @@
|
|||
package azure
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/options"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
// Config contains all configuration necessary to connect to an azure compatible
|
||||
|
@ -33,7 +35,7 @@ func init() {
|
|||
|
||||
// ParseConfig parses the string s and extracts the azure config. The
|
||||
// configuration format is azure:containerName:/[prefix].
|
||||
func ParseConfig(s string) (interface{}, error) {
|
||||
func ParseConfig(s string) (*Config, error) {
|
||||
if !strings.HasPrefix(s, "azure:") {
|
||||
return nil, errors.New("azure: invalid format")
|
||||
}
|
||||
|
@ -51,5 +53,23 @@ func ParseConfig(s string) (interface{}, error) {
|
|||
cfg := NewConfig()
|
||||
cfg.Container = container
|
||||
cfg.Prefix = prefix
|
||||
return cfg, nil
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
var _ restic.ApplyEnvironmenter = &Config{}
|
||||
|
||||
// ApplyEnvironment saves values from the environment to the config.
|
||||
func (cfg *Config) ApplyEnvironment(prefix string) error {
|
||||
if cfg.AccountName == "" {
|
||||
cfg.AccountName = os.Getenv(prefix + "AZURE_ACCOUNT_NAME")
|
||||
}
|
||||
|
||||
if cfg.AccountKey.String() == "" {
|
||||
cfg.AccountKey = options.NewSecretString(os.Getenv(prefix + "AZURE_ACCOUNT_KEY"))
|
||||
}
|
||||
|
||||
if cfg.AccountSAS.String() == "" {
|
||||
cfg.AccountSAS = options.NewSecretString(os.Getenv(prefix + "AZURE_ACCOUNT_SAS"))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -1,22 +1,23 @@
|
|||
package azure
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"testing"
|
||||
|
||||
var configTests = []struct {
|
||||
s string
|
||||
cfg Config
|
||||
}{
|
||||
{"azure:container-name:/", Config{
|
||||
"github.com/restic/restic/internal/backend/test"
|
||||
)
|
||||
|
||||
var configTests = []test.ConfigTestData[Config]{
|
||||
{S: "azure:container-name:/", Cfg: Config{
|
||||
Container: "container-name",
|
||||
Prefix: "",
|
||||
Connections: 5,
|
||||
}},
|
||||
{"azure:container-name:/prefix/directory", Config{
|
||||
{S: "azure:container-name:/prefix/directory", Cfg: Config{
|
||||
Container: "container-name",
|
||||
Prefix: "prefix/directory",
|
||||
Connections: 5,
|
||||
}},
|
||||
{"azure:container-name:/prefix/directory/", Config{
|
||||
{S: "azure:container-name:/prefix/directory/", Cfg: Config{
|
||||
Container: "container-name",
|
||||
Prefix: "prefix/directory",
|
||||
Connections: 5,
|
||||
|
@ -24,17 +25,5 @@ var configTests = []struct {
|
|||
}
|
||||
|
||||
func TestParseConfig(t *testing.T) {
|
||||
for i, test := range configTests {
|
||||
cfg, err := ParseConfig(test.s)
|
||||
if err != nil {
|
||||
t.Errorf("test %d:%s failed: %v", i, test.s, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if cfg != test.cfg {
|
||||
t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v",
|
||||
i, test.s, test.cfg, cfg)
|
||||
continue
|
||||
}
|
||||
}
|
||||
test.ParseConfigTester(t, ParseConfig, configTests)
|
||||
}
|
||||
|
|
|
@ -10,19 +10,18 @@ import (
|
|||
"github.com/restic/restic/internal/backend"
|
||||
"github.com/restic/restic/internal/backend/b2"
|
||||
"github.com/restic/restic/internal/backend/test"
|
||||
"github.com/restic/restic/internal/options"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
func newB2TestSuite(t testing.TB) *test.Suite {
|
||||
func newB2TestSuite(t testing.TB) *test.Suite[b2.Config] {
|
||||
tr, err := backend.Transport(backend.TransportOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("cannot create transport for tests: %v", err)
|
||||
}
|
||||
|
||||
return &test.Suite{
|
||||
return &test.Suite[b2.Config]{
|
||||
// do not use excessive data
|
||||
MinimalData: true,
|
||||
|
||||
|
@ -30,34 +29,33 @@ func newB2TestSuite(t testing.TB) *test.Suite {
|
|||
WaitForDelayedRemoval: 10 * time.Second,
|
||||
|
||||
// NewConfig returns a config for a new temporary backend that will be used in tests.
|
||||
NewConfig: func() (interface{}, error) {
|
||||
b2cfg, err := b2.ParseConfig(os.Getenv("RESTIC_TEST_B2_REPOSITORY"))
|
||||
NewConfig: func() (*b2.Config, error) {
|
||||
cfg, err := b2.ParseConfig(os.Getenv("RESTIC_TEST_B2_REPOSITORY"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = cfg.ApplyEnvironment("RESTIC_TEST_")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg := b2cfg.(b2.Config)
|
||||
cfg.AccountID = os.Getenv("RESTIC_TEST_B2_ACCOUNT_ID")
|
||||
cfg.Key = options.NewSecretString(os.Getenv("RESTIC_TEST_B2_ACCOUNT_KEY"))
|
||||
cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
|
||||
return cfg, nil
|
||||
},
|
||||
|
||||
// CreateFn is a function that creates a temporary repository for the tests.
|
||||
Create: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(b2.Config)
|
||||
Create: func(cfg b2.Config) (restic.Backend, error) {
|
||||
return b2.Create(context.Background(), cfg, tr)
|
||||
},
|
||||
|
||||
// OpenFn is a function that opens a previously created temporary repository.
|
||||
Open: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(b2.Config)
|
||||
Open: func(cfg b2.Config) (restic.Backend, error) {
|
||||
return b2.Open(context.Background(), cfg, tr)
|
||||
},
|
||||
|
||||
// CleanupFn removes data created during the tests.
|
||||
Cleanup: func(config interface{}) error {
|
||||
cfg := config.(b2.Config)
|
||||
Cleanup: func(cfg b2.Config) error {
|
||||
be, err := b2.Open(context.Background(), cfg, tr)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
@ -1,12 +1,14 @@
|
|||
package b2
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/options"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
// Config contains all configuration necessary to connect to an b2 compatible
|
||||
|
@ -58,7 +60,7 @@ func checkBucketName(name string) error {
|
|||
// ParseConfig parses the string s and extracts the b2 config. The supported
|
||||
// configuration format is b2:bucketname/prefix. If no prefix is given the
|
||||
// prefix "restic" will be used.
|
||||
func ParseConfig(s string) (interface{}, error) {
|
||||
func ParseConfig(s string) (*Config, error) {
|
||||
if !strings.HasPrefix(s, "b2:") {
|
||||
return nil, errors.New("invalid format, want: b2:bucket-name[:path]")
|
||||
}
|
||||
|
@ -77,5 +79,27 @@ func ParseConfig(s string) (interface{}, error) {
|
|||
cfg.Bucket = bucket
|
||||
cfg.Prefix = prefix
|
||||
|
||||
return cfg, nil
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
var _ restic.ApplyEnvironmenter = &Config{}
|
||||
|
||||
// ApplyEnvironment saves values from the environment to the config.
|
||||
func (cfg *Config) ApplyEnvironment(prefix string) error {
|
||||
if cfg.AccountID == "" {
|
||||
cfg.AccountID = os.Getenv(prefix + "B2_ACCOUNT_ID")
|
||||
}
|
||||
|
||||
if cfg.AccountID == "" {
|
||||
return errors.Fatalf("unable to open B2 backend: Account ID ($B2_ACCOUNT_ID) is empty")
|
||||
}
|
||||
|
||||
if cfg.Key.String() == "" {
|
||||
cfg.Key = options.NewSecretString(os.Getenv(prefix + "B2_ACCOUNT_KEY"))
|
||||
}
|
||||
|
||||
if cfg.Key.String() == "" {
|
||||
return errors.Fatalf("unable to open B2 backend: Key ($B2_ACCOUNT_KEY) is empty")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -1,37 +1,38 @@
|
|||
package b2
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"testing"
|
||||
|
||||
var configTests = []struct {
|
||||
s string
|
||||
cfg Config
|
||||
}{
|
||||
{"b2:bucketname", Config{
|
||||
"github.com/restic/restic/internal/backend/test"
|
||||
)
|
||||
|
||||
var configTests = []test.ConfigTestData[Config]{
|
||||
{S: "b2:bucketname", Cfg: Config{
|
||||
Bucket: "bucketname",
|
||||
Prefix: "",
|
||||
Connections: 5,
|
||||
}},
|
||||
{"b2:bucketname:", Config{
|
||||
{S: "b2:bucketname:", Cfg: Config{
|
||||
Bucket: "bucketname",
|
||||
Prefix: "",
|
||||
Connections: 5,
|
||||
}},
|
||||
{"b2:bucketname:/prefix/directory", Config{
|
||||
{S: "b2:bucketname:/prefix/directory", Cfg: Config{
|
||||
Bucket: "bucketname",
|
||||
Prefix: "prefix/directory",
|
||||
Connections: 5,
|
||||
}},
|
||||
{"b2:foobar", Config{
|
||||
{S: "b2:foobar", Cfg: Config{
|
||||
Bucket: "foobar",
|
||||
Prefix: "",
|
||||
Connections: 5,
|
||||
}},
|
||||
{"b2:foobar:", Config{
|
||||
{S: "b2:foobar:", Cfg: Config{
|
||||
Bucket: "foobar",
|
||||
Prefix: "",
|
||||
Connections: 5,
|
||||
}},
|
||||
{"b2:foobar:/", Config{
|
||||
{S: "b2:foobar:/", Cfg: Config{
|
||||
Bucket: "foobar",
|
||||
Prefix: "",
|
||||
Connections: 5,
|
||||
|
@ -39,19 +40,7 @@ var configTests = []struct {
|
|||
}
|
||||
|
||||
func TestParseConfig(t *testing.T) {
|
||||
for _, test := range configTests {
|
||||
t.Run("", func(t *testing.T) {
|
||||
cfg, err := ParseConfig(test.s)
|
||||
if err != nil {
|
||||
t.Fatalf("%s failed: %v", test.s, err)
|
||||
}
|
||||
|
||||
if cfg != test.cfg {
|
||||
t.Fatalf("input: %s\n wrong config, want:\n %#v\ngot:\n %#v",
|
||||
test.s, test.cfg, cfg)
|
||||
}
|
||||
})
|
||||
}
|
||||
test.ParseConfigTester(t, ParseConfig, configTests)
|
||||
}
|
||||
|
||||
var invalidConfigTests = []struct {
|
||||
|
|
|
@ -1,11 +1,13 @@
|
|||
package gs
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/options"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
// Config contains all configuration necessary to connect to a Google Cloud Storage
|
||||
|
@ -34,7 +36,7 @@ func init() {
|
|||
|
||||
// ParseConfig parses the string s and extracts the gcs config. The
|
||||
// supported configuration format is gs:bucketName:/[prefix].
|
||||
func ParseConfig(s string) (interface{}, error) {
|
||||
func ParseConfig(s string) (*Config, error) {
|
||||
if !strings.HasPrefix(s, "gs:") {
|
||||
return nil, errors.New("gs: invalid format")
|
||||
}
|
||||
|
@ -54,5 +56,15 @@ func ParseConfig(s string) (interface{}, error) {
|
|||
cfg := NewConfig()
|
||||
cfg.Bucket = bucket
|
||||
cfg.Prefix = prefix
|
||||
return cfg, nil
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
var _ restic.ApplyEnvironmenter = &Config{}
|
||||
|
||||
// ApplyEnvironment saves values from the environment to the config.
|
||||
func (cfg *Config) ApplyEnvironment(prefix string) error {
|
||||
if cfg.ProjectID == "" {
|
||||
cfg.ProjectID = os.Getenv(prefix + "GOOGLE_PROJECT_ID")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -1,24 +1,25 @@
|
|||
package gs
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"testing"
|
||||
|
||||
var configTests = []struct {
|
||||
s string
|
||||
cfg Config
|
||||
}{
|
||||
{"gs:bucketname:/", Config{
|
||||
"github.com/restic/restic/internal/backend/test"
|
||||
)
|
||||
|
||||
var configTests = []test.ConfigTestData[Config]{
|
||||
{S: "gs:bucketname:/", Cfg: Config{
|
||||
Bucket: "bucketname",
|
||||
Prefix: "",
|
||||
Connections: 5,
|
||||
Region: "us",
|
||||
}},
|
||||
{"gs:bucketname:/prefix/directory", Config{
|
||||
{S: "gs:bucketname:/prefix/directory", Cfg: Config{
|
||||
Bucket: "bucketname",
|
||||
Prefix: "prefix/directory",
|
||||
Connections: 5,
|
||||
Region: "us",
|
||||
}},
|
||||
{"gs:bucketname:/prefix/directory/", Config{
|
||||
{S: "gs:bucketname:/prefix/directory/", Cfg: Config{
|
||||
Bucket: "bucketname",
|
||||
Prefix: "prefix/directory",
|
||||
Connections: 5,
|
||||
|
@ -27,17 +28,5 @@ var configTests = []struct {
|
|||
}
|
||||
|
||||
func TestParseConfig(t *testing.T) {
|
||||
for i, test := range configTests {
|
||||
cfg, err := ParseConfig(test.s)
|
||||
if err != nil {
|
||||
t.Errorf("test %d:%s failed: %v", i, test.s, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if cfg != test.cfg {
|
||||
t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v",
|
||||
i, test.s, test.cfg, cfg)
|
||||
continue
|
||||
}
|
||||
}
|
||||
test.ParseConfigTester(t, ParseConfig, configTests)
|
||||
}
|
||||
|
|
|
@ -15,33 +15,30 @@ import (
|
|||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
func newGSTestSuite(t testing.TB) *test.Suite {
|
||||
func newGSTestSuite(t testing.TB) *test.Suite[gs.Config] {
|
||||
tr, err := backend.Transport(backend.TransportOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("cannot create transport for tests: %v", err)
|
||||
}
|
||||
|
||||
return &test.Suite{
|
||||
return &test.Suite[gs.Config]{
|
||||
// do not use excessive data
|
||||
MinimalData: true,
|
||||
|
||||
// NewConfig returns a config for a new temporary backend that will be used in tests.
|
||||
NewConfig: func() (interface{}, error) {
|
||||
gscfg, err := gs.ParseConfig(os.Getenv("RESTIC_TEST_GS_REPOSITORY"))
|
||||
NewConfig: func() (*gs.Config, error) {
|
||||
cfg, err := gs.ParseConfig(os.Getenv("RESTIC_TEST_GS_REPOSITORY"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg := gscfg.(gs.Config)
|
||||
cfg.ProjectID = os.Getenv("RESTIC_TEST_GS_PROJECT_ID")
|
||||
cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
|
||||
return cfg, nil
|
||||
},
|
||||
|
||||
// CreateFn is a function that creates a temporary repository for the tests.
|
||||
Create: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(gs.Config)
|
||||
|
||||
Create: func(cfg gs.Config) (restic.Backend, error) {
|
||||
be, err := gs.Create(context.Background(), cfg, tr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
@ -60,15 +57,12 @@ func newGSTestSuite(t testing.TB) *test.Suite {
|
|||
},
|
||||
|
||||
// OpenFn is a function that opens a previously created temporary repository.
|
||||
Open: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(gs.Config)
|
||||
Open: func(cfg gs.Config) (restic.Backend, error) {
|
||||
return gs.Open(cfg, tr)
|
||||
},
|
||||
|
||||
// CleanupFn removes data created during the tests.
|
||||
Cleanup: func(config interface{}) error {
|
||||
cfg := config.(gs.Config)
|
||||
|
||||
Cleanup: func(cfg gs.Config) error {
|
||||
be, err := gs.Open(cfg, tr)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
@ -27,12 +27,12 @@ func init() {
|
|||
}
|
||||
|
||||
// ParseConfig parses a local backend config.
|
||||
func ParseConfig(s string) (interface{}, error) {
|
||||
func ParseConfig(s string) (*Config, error) {
|
||||
if !strings.HasPrefix(s, "local:") {
|
||||
return nil, errors.New(`invalid format, prefix "local" not found`)
|
||||
}
|
||||
|
||||
cfg := NewConfig()
|
||||
cfg.Path = s[6:]
|
||||
return cfg, nil
|
||||
return &cfg, nil
|
||||
}
|
||||
|
|
18
internal/backend/local/config_test.go
Normal file
18
internal/backend/local/config_test.go
Normal file
|
@ -0,0 +1,18 @@
|
|||
package local
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/backend/test"
|
||||
)
|
||||
|
||||
var configTests = []test.ConfigTestData[Config]{
|
||||
{S: "local:/some/path", Cfg: Config{
|
||||
Path: "/some/path",
|
||||
Connections: 2,
|
||||
}},
|
||||
}
|
||||
|
||||
func TestParseConfig(t *testing.T) {
|
||||
test.ParseConfigTester(t, ParseConfig, configTests)
|
||||
}
|
|
@ -148,6 +148,13 @@ func (b *Local) Save(_ context.Context, h restic.Handle, rd restic.RewindReader)
|
|||
}
|
||||
}(f)
|
||||
|
||||
// preallocate disk space
|
||||
if size := rd.Length(); size > 0 {
|
||||
if err := fs.PreallocateFile(f, size); err != nil {
|
||||
debug.Log("Failed to preallocate %v with size %v: %v", finalname, size, err)
|
||||
}
|
||||
}
|
||||
|
||||
// save data, then sync
|
||||
wbytes, err := io.Copy(f, rd)
|
||||
if err != nil {
|
||||
|
|
|
@ -12,10 +12,10 @@ import (
|
|||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
func newTestSuite(t testing.TB) *test.Suite {
|
||||
return &test.Suite{
|
||||
func newTestSuite(t testing.TB) *test.Suite[local.Config] {
|
||||
return &test.Suite[local.Config]{
|
||||
// NewConfig returns a config for a new temporary backend that will be used in tests.
|
||||
NewConfig: func() (interface{}, error) {
|
||||
NewConfig: func() (*local.Config, error) {
|
||||
dir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-local-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
@ -23,7 +23,7 @@ func newTestSuite(t testing.TB) *test.Suite {
|
|||
|
||||
t.Logf("create new backend at %v", dir)
|
||||
|
||||
cfg := local.Config{
|
||||
cfg := &local.Config{
|
||||
Path: dir,
|
||||
Connections: 2,
|
||||
}
|
||||
|
@ -31,20 +31,17 @@ func newTestSuite(t testing.TB) *test.Suite {
|
|||
},
|
||||
|
||||
// CreateFn is a function that creates a temporary repository for the tests.
|
||||
Create: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(local.Config)
|
||||
Create: func(cfg local.Config) (restic.Backend, error) {
|
||||
return local.Create(context.TODO(), cfg)
|
||||
},
|
||||
|
||||
// OpenFn is a function that opens a previously created temporary repository.
|
||||
Open: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(local.Config)
|
||||
Open: func(cfg local.Config) (restic.Backend, error) {
|
||||
return local.Open(context.TODO(), cfg)
|
||||
},
|
||||
|
||||
// CleanupFn removes data created during the tests.
|
||||
Cleanup: func(config interface{}) error {
|
||||
cfg := config.(local.Config)
|
||||
Cleanup: func(cfg local.Config) error {
|
||||
if !rtest.TestCleanupTempDirs {
|
||||
t.Logf("leaving test backend dir at %v", cfg.Path)
|
||||
}
|
||||
|
|
|
@ -30,19 +30,25 @@ type parser struct {
|
|||
stripPassword func(string) string
|
||||
}
|
||||
|
||||
func configToAny[C any](parser func(string) (*C, error)) func(string) (interface{}, error) {
|
||||
return func(s string) (interface{}, error) {
|
||||
return parser(s)
|
||||
}
|
||||
}
|
||||
|
||||
// parsers is a list of valid config parsers for the backends. The first parser
|
||||
// is the fallback and should always be set to the local backend.
|
||||
var parsers = []parser{
|
||||
{"b2", b2.ParseConfig, noPassword},
|
||||
{"local", local.ParseConfig, noPassword},
|
||||
{"sftp", sftp.ParseConfig, noPassword},
|
||||
{"s3", s3.ParseConfig, noPassword},
|
||||
{"gs", gs.ParseConfig, noPassword},
|
||||
{"azure", azure.ParseConfig, noPassword},
|
||||
{"swift", swift.ParseConfig, noPassword},
|
||||
{"rest", rest.ParseConfig, rest.StripPassword},
|
||||
{"rclone", rclone.ParseConfig, noPassword},
|
||||
{"smb", smb.ParseConfig, noPassword},
|
||||
{"b2", configToAny(b2.ParseConfig), noPassword},
|
||||
{"local", configToAny(local.ParseConfig), noPassword},
|
||||
{"sftp", configToAny(sftp.ParseConfig), noPassword},
|
||||
{"s3", configToAny(s3.ParseConfig), noPassword},
|
||||
{"gs", configToAny(gs.ParseConfig), noPassword},
|
||||
{"azure", configToAny(azure.ParseConfig), noPassword},
|
||||
{"swift", configToAny(swift.ParseConfig), noPassword},
|
||||
{"rest", configToAny(rest.ParseConfig), rest.StripPassword},
|
||||
{"rclone", configToAny(rclone.ParseConfig), noPassword},
|
||||
{"smb", configToAny(smb.ParseConfig), noPassword},
|
||||
}
|
||||
|
||||
// noPassword returns the repository location unchanged (there's no sensitive information there)
|
||||
|
|
|
@ -29,7 +29,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"local:/srv/repo",
|
||||
Location{Scheme: "local",
|
||||
Config: local.Config{
|
||||
Config: &local.Config{
|
||||
Path: "/srv/repo",
|
||||
Connections: 2,
|
||||
},
|
||||
|
@ -38,7 +38,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"local:dir1/dir2",
|
||||
Location{Scheme: "local",
|
||||
Config: local.Config{
|
||||
Config: &local.Config{
|
||||
Path: "dir1/dir2",
|
||||
Connections: 2,
|
||||
},
|
||||
|
@ -47,7 +47,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"local:dir1/dir2",
|
||||
Location{Scheme: "local",
|
||||
Config: local.Config{
|
||||
Config: &local.Config{
|
||||
Path: "dir1/dir2",
|
||||
Connections: 2,
|
||||
},
|
||||
|
@ -56,7 +56,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"dir1/dir2",
|
||||
Location{Scheme: "local",
|
||||
Config: local.Config{
|
||||
Config: &local.Config{
|
||||
Path: "dir1/dir2",
|
||||
Connections: 2,
|
||||
},
|
||||
|
@ -65,7 +65,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"/dir1/dir2",
|
||||
Location{Scheme: "local",
|
||||
Config: local.Config{
|
||||
Config: &local.Config{
|
||||
Path: "/dir1/dir2",
|
||||
Connections: 2,
|
||||
},
|
||||
|
@ -74,7 +74,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"local:../dir1/dir2",
|
||||
Location{Scheme: "local",
|
||||
Config: local.Config{
|
||||
Config: &local.Config{
|
||||
Path: "../dir1/dir2",
|
||||
Connections: 2,
|
||||
},
|
||||
|
@ -83,7 +83,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"/dir1/dir2",
|
||||
Location{Scheme: "local",
|
||||
Config: local.Config{
|
||||
Config: &local.Config{
|
||||
Path: "/dir1/dir2",
|
||||
Connections: 2,
|
||||
},
|
||||
|
@ -92,7 +92,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"/dir1:foobar/dir2",
|
||||
Location{Scheme: "local",
|
||||
Config: local.Config{
|
||||
Config: &local.Config{
|
||||
Path: "/dir1:foobar/dir2",
|
||||
Connections: 2,
|
||||
},
|
||||
|
@ -101,7 +101,7 @@ var parseTests = []struct {
|
|||
{
|
||||
`\dir1\foobar\dir2`,
|
||||
Location{Scheme: "local",
|
||||
Config: local.Config{
|
||||
Config: &local.Config{
|
||||
Path: `\dir1\foobar\dir2`,
|
||||
Connections: 2,
|
||||
},
|
||||
|
@ -110,7 +110,7 @@ var parseTests = []struct {
|
|||
{
|
||||
`c:\dir1\foobar\dir2`,
|
||||
Location{Scheme: "local",
|
||||
Config: local.Config{
|
||||
Config: &local.Config{
|
||||
Path: `c:\dir1\foobar\dir2`,
|
||||
Connections: 2,
|
||||
},
|
||||
|
@ -119,7 +119,7 @@ var parseTests = []struct {
|
|||
{
|
||||
`C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`,
|
||||
Location{Scheme: "local",
|
||||
Config: local.Config{
|
||||
Config: &local.Config{
|
||||
Path: `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`,
|
||||
Connections: 2,
|
||||
},
|
||||
|
@ -128,7 +128,7 @@ var parseTests = []struct {
|
|||
{
|
||||
`c:/dir1/foobar/dir2`,
|
||||
Location{Scheme: "local",
|
||||
Config: local.Config{
|
||||
Config: &local.Config{
|
||||
Path: `c:/dir1/foobar/dir2`,
|
||||
Connections: 2,
|
||||
},
|
||||
|
@ -137,7 +137,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"sftp:user@host:/srv/repo",
|
||||
Location{Scheme: "sftp",
|
||||
Config: sftp.Config{
|
||||
Config: &sftp.Config{
|
||||
User: "user",
|
||||
Host: "host",
|
||||
Path: "/srv/repo",
|
||||
|
@ -148,7 +148,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"sftp:host:/srv/repo",
|
||||
Location{Scheme: "sftp",
|
||||
Config: sftp.Config{
|
||||
Config: &sftp.Config{
|
||||
User: "",
|
||||
Host: "host",
|
||||
Path: "/srv/repo",
|
||||
|
@ -159,7 +159,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"sftp://user@host/srv/repo",
|
||||
Location{Scheme: "sftp",
|
||||
Config: sftp.Config{
|
||||
Config: &sftp.Config{
|
||||
User: "user",
|
||||
Host: "host",
|
||||
Path: "srv/repo",
|
||||
|
@ -170,7 +170,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"sftp://user@host//srv/repo",
|
||||
Location{Scheme: "sftp",
|
||||
Config: sftp.Config{
|
||||
Config: &sftp.Config{
|
||||
User: "user",
|
||||
Host: "host",
|
||||
Path: "/srv/repo",
|
||||
|
@ -182,7 +182,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"s3://eu-central-1/bucketname",
|
||||
Location{Scheme: "s3",
|
||||
Config: s3.Config{
|
||||
Config: &s3.Config{
|
||||
Endpoint: "eu-central-1",
|
||||
Bucket: "bucketname",
|
||||
Prefix: "",
|
||||
|
@ -193,7 +193,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"s3://hostname.foo/bucketname",
|
||||
Location{Scheme: "s3",
|
||||
Config: s3.Config{
|
||||
Config: &s3.Config{
|
||||
Endpoint: "hostname.foo",
|
||||
Bucket: "bucketname",
|
||||
Prefix: "",
|
||||
|
@ -204,7 +204,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"s3://hostname.foo/bucketname/prefix/directory",
|
||||
Location{Scheme: "s3",
|
||||
Config: s3.Config{
|
||||
Config: &s3.Config{
|
||||
Endpoint: "hostname.foo",
|
||||
Bucket: "bucketname",
|
||||
Prefix: "prefix/directory",
|
||||
|
@ -215,7 +215,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"s3:eu-central-1/repo",
|
||||
Location{Scheme: "s3",
|
||||
Config: s3.Config{
|
||||
Config: &s3.Config{
|
||||
Endpoint: "eu-central-1",
|
||||
Bucket: "repo",
|
||||
Prefix: "",
|
||||
|
@ -226,7 +226,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"s3:eu-central-1/repo/prefix/directory",
|
||||
Location{Scheme: "s3",
|
||||
Config: s3.Config{
|
||||
Config: &s3.Config{
|
||||
Endpoint: "eu-central-1",
|
||||
Bucket: "repo",
|
||||
Prefix: "prefix/directory",
|
||||
|
@ -237,7 +237,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"s3:https://hostname.foo/repo",
|
||||
Location{Scheme: "s3",
|
||||
Config: s3.Config{
|
||||
Config: &s3.Config{
|
||||
Endpoint: "hostname.foo",
|
||||
Bucket: "repo",
|
||||
Prefix: "",
|
||||
|
@ -248,7 +248,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"s3:https://hostname.foo/repo/prefix/directory",
|
||||
Location{Scheme: "s3",
|
||||
Config: s3.Config{
|
||||
Config: &s3.Config{
|
||||
Endpoint: "hostname.foo",
|
||||
Bucket: "repo",
|
||||
Prefix: "prefix/directory",
|
||||
|
@ -259,7 +259,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"s3:http://hostname.foo/repo",
|
||||
Location{Scheme: "s3",
|
||||
Config: s3.Config{
|
||||
Config: &s3.Config{
|
||||
Endpoint: "hostname.foo",
|
||||
Bucket: "repo",
|
||||
Prefix: "",
|
||||
|
@ -271,7 +271,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"swift:container17:/",
|
||||
Location{Scheme: "swift",
|
||||
Config: swift.Config{
|
||||
Config: &swift.Config{
|
||||
Container: "container17",
|
||||
Prefix: "",
|
||||
Connections: 5,
|
||||
|
@ -281,7 +281,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"swift:container17:/prefix97",
|
||||
Location{Scheme: "swift",
|
||||
Config: swift.Config{
|
||||
Config: &swift.Config{
|
||||
Container: "container17",
|
||||
Prefix: "prefix97",
|
||||
Connections: 5,
|
||||
|
@ -291,7 +291,7 @@ var parseTests = []struct {
|
|||
{
|
||||
"rest:http://hostname.foo:1234/",
|
||||
Location{Scheme: "rest",
|
||||
Config: rest.Config{
|
||||
Config: &rest.Config{
|
||||
URL: parseURL("http://hostname.foo:1234/"),
|
||||
Connections: 5,
|
||||
},
|
||||
|
@ -299,7 +299,7 @@ var parseTests = []struct {
|
|||
},
|
||||
{
|
||||
"b2:bucketname:/prefix", Location{Scheme: "b2",
|
||||
Config: b2.Config{
|
||||
Config: &b2.Config{
|
||||
Bucket: "bucketname",
|
||||
Prefix: "prefix",
|
||||
Connections: 5,
|
||||
|
@ -308,7 +308,7 @@ var parseTests = []struct {
|
|||
},
|
||||
{
|
||||
"b2:bucketname", Location{Scheme: "b2",
|
||||
Config: b2.Config{
|
||||
Config: &b2.Config{
|
||||
Bucket: "bucketname",
|
||||
Prefix: "",
|
||||
Connections: 5,
|
||||
|
|
|
@ -15,19 +15,19 @@ type memConfig struct {
|
|||
be restic.Backend
|
||||
}
|
||||
|
||||
func newTestSuite() *test.Suite {
|
||||
return &test.Suite{
|
||||
func newTestSuite() *test.Suite[*memConfig] {
|
||||
return &test.Suite[*memConfig]{
|
||||
// NewConfig returns a config for a new temporary backend that will be used in tests.
|
||||
NewConfig: func() (interface{}, error) {
|
||||
return &memConfig{}, nil
|
||||
NewConfig: func() (**memConfig, error) {
|
||||
cfg := &memConfig{}
|
||||
return &cfg, nil
|
||||
},
|
||||
|
||||
// CreateFn is a function that creates a temporary repository for the tests.
|
||||
Create: func(cfg interface{}) (restic.Backend, error) {
|
||||
c := cfg.(*memConfig)
|
||||
if c.be != nil {
|
||||
_, err := c.be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
|
||||
if err != nil && !c.be.IsNotExist(err) {
|
||||
Create: func(cfg *memConfig) (restic.Backend, error) {
|
||||
if cfg.be != nil {
|
||||
_, err := cfg.be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
|
||||
if err != nil && !cfg.be.IsNotExist(err) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
@ -36,21 +36,20 @@ func newTestSuite() *test.Suite {
|
|||
}
|
||||
}
|
||||
|
||||
c.be = mem.New()
|
||||
return c.be, nil
|
||||
cfg.be = mem.New()
|
||||
return cfg.be, nil
|
||||
},
|
||||
|
||||
// OpenFn is a function that opens a previously created temporary repository.
|
||||
Open: func(cfg interface{}) (restic.Backend, error) {
|
||||
c := cfg.(*memConfig)
|
||||
if c.be == nil {
|
||||
c.be = mem.New()
|
||||
Open: func(cfg *memConfig) (restic.Backend, error) {
|
||||
if cfg.be == nil {
|
||||
cfg.be = mem.New()
|
||||
}
|
||||
return c.be, nil
|
||||
return cfg.be, nil
|
||||
},
|
||||
|
||||
// CleanupFn removes data created during the tests.
|
||||
Cleanup: func(cfg interface{}) error {
|
||||
Cleanup: func(cfg *memConfig) error {
|
||||
// no cleanup needed
|
||||
return nil
|
||||
},
|
||||
|
|
|
@ -12,22 +12,21 @@ import (
|
|||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
func newTestSuite(t testing.TB) *test.Suite {
|
||||
func newTestSuite(t testing.TB) *test.Suite[rclone.Config] {
|
||||
dir := rtest.TempDir(t)
|
||||
|
||||
return &test.Suite{
|
||||
return &test.Suite[rclone.Config]{
|
||||
// NewConfig returns a config for a new temporary backend that will be used in tests.
|
||||
NewConfig: func() (interface{}, error) {
|
||||
NewConfig: func() (*rclone.Config, error) {
|
||||
t.Logf("use backend at %v", dir)
|
||||
cfg := rclone.NewConfig()
|
||||
cfg.Remote = dir
|
||||
return cfg, nil
|
||||
return &cfg, nil
|
||||
},
|
||||
|
||||
// CreateFn is a function that creates a temporary repository for the tests.
|
||||
Create: func(config interface{}) (restic.Backend, error) {
|
||||
Create: func(cfg rclone.Config) (restic.Backend, error) {
|
||||
t.Logf("Create()")
|
||||
cfg := config.(rclone.Config)
|
||||
be, err := rclone.Create(context.TODO(), cfg)
|
||||
var e *exec.Error
|
||||
if errors.As(err, &e) && e.Err == exec.ErrNotFound {
|
||||
|
@ -38,9 +37,8 @@ func newTestSuite(t testing.TB) *test.Suite {
|
|||
},
|
||||
|
||||
// OpenFn is a function that opens a previously created temporary repository.
|
||||
Open: func(config interface{}) (restic.Backend, error) {
|
||||
Open: func(cfg rclone.Config) (restic.Backend, error) {
|
||||
t.Logf("Open()")
|
||||
cfg := config.(rclone.Config)
|
||||
return rclone.Open(cfg, nil)
|
||||
},
|
||||
}
|
||||
|
|
|
@ -34,7 +34,7 @@ func NewConfig() Config {
|
|||
}
|
||||
|
||||
// ParseConfig parses the string s and extracts the remote server URL.
|
||||
func ParseConfig(s string) (interface{}, error) {
|
||||
func ParseConfig(s string) (*Config, error) {
|
||||
if !strings.HasPrefix(s, "rclone:") {
|
||||
return nil, errors.New("invalid rclone backend specification")
|
||||
}
|
||||
|
@ -42,5 +42,5 @@ func ParseConfig(s string) (interface{}, error) {
|
|||
s = s[7:]
|
||||
cfg := NewConfig()
|
||||
cfg.Remote = s
|
||||
return cfg, nil
|
||||
return &cfg, nil
|
||||
}
|
||||
|
|
|
@ -1,18 +1,15 @@
|
|||
package rclone
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/backend/test"
|
||||
)
|
||||
|
||||
func TestParseConfig(t *testing.T) {
|
||||
var tests = []struct {
|
||||
s string
|
||||
cfg Config
|
||||
}{
|
||||
var configTests = []test.ConfigTestData[Config]{
|
||||
{
|
||||
"rclone:local:foo:/bar",
|
||||
Config{
|
||||
S: "rclone:local:foo:/bar",
|
||||
Cfg: Config{
|
||||
Remote: "local:foo:/bar",
|
||||
Program: defaultConfig.Program,
|
||||
Args: defaultConfig.Args,
|
||||
|
@ -22,16 +19,6 @@ func TestParseConfig(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run("", func(t *testing.T) {
|
||||
cfg, err := ParseConfig(test.s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(cfg, test.cfg) {
|
||||
t.Fatalf("wrong config, want:\n %v\ngot:\n %v", test.cfg, cfg)
|
||||
}
|
||||
})
|
||||
}
|
||||
func TestParseConfig(t *testing.T) {
|
||||
test.ParseConfigTester(t, ParseConfig, configTests)
|
||||
}
|
||||
|
|
|
@ -26,7 +26,7 @@ func NewConfig() Config {
|
|||
}
|
||||
|
||||
// ParseConfig parses the string s and extracts the REST server URL.
|
||||
func ParseConfig(s string) (interface{}, error) {
|
||||
func ParseConfig(s string) (*Config, error) {
|
||||
if !strings.HasPrefix(s, "rest:") {
|
||||
return nil, errors.New("invalid REST backend specification")
|
||||
}
|
||||
|
@ -40,7 +40,7 @@ func ParseConfig(s string) (interface{}, error) {
|
|||
|
||||
cfg := NewConfig()
|
||||
cfg.URL = u
|
||||
return cfg, nil
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
// StripPassword removes the password from the URL
|
||||
|
|
|
@ -2,8 +2,9 @@ package rest
|
|||
|
||||
import (
|
||||
"net/url"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/backend/test"
|
||||
)
|
||||
|
||||
func parseURL(s string) *url.URL {
|
||||
|
@ -15,20 +16,17 @@ func parseURL(s string) *url.URL {
|
|||
return u
|
||||
}
|
||||
|
||||
var configTests = []struct {
|
||||
s string
|
||||
cfg Config
|
||||
}{
|
||||
var configTests = []test.ConfigTestData[Config]{
|
||||
{
|
||||
s: "rest:http://localhost:1234",
|
||||
cfg: Config{
|
||||
S: "rest:http://localhost:1234",
|
||||
Cfg: Config{
|
||||
URL: parseURL("http://localhost:1234/"),
|
||||
Connections: 5,
|
||||
},
|
||||
},
|
||||
{
|
||||
s: "rest:http://localhost:1234/",
|
||||
cfg: Config{
|
||||
S: "rest:http://localhost:1234/",
|
||||
Cfg: Config{
|
||||
URL: parseURL("http://localhost:1234/"),
|
||||
Connections: 5,
|
||||
},
|
||||
|
@ -36,17 +34,5 @@ var configTests = []struct {
|
|||
}
|
||||
|
||||
func TestParseConfig(t *testing.T) {
|
||||
for _, test := range configTests {
|
||||
t.Run("", func(t *testing.T) {
|
||||
cfg, err := ParseConfig(test.s)
|
||||
if err != nil {
|
||||
t.Fatalf("%s failed: %v", test.s, err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(cfg, test.cfg) {
|
||||
t.Fatalf("\ninput: %s\n wrong config, want:\n %v\ngot:\n %v",
|
||||
test.s, test.cfg, cfg)
|
||||
}
|
||||
})
|
||||
}
|
||||
test.ParseConfigTester(t, ParseConfig, configTests)
|
||||
}
|
||||
|
|
|
@ -67,36 +67,34 @@ func runRESTServer(ctx context.Context, t testing.TB, dir string) (*url.URL, fun
|
|||
return url, cleanup
|
||||
}
|
||||
|
||||
func newTestSuite(_ context.Context, t testing.TB, url *url.URL, minimalData bool) *test.Suite {
|
||||
func newTestSuite(_ context.Context, t testing.TB, url *url.URL, minimalData bool) *test.Suite[rest.Config] {
|
||||
tr, err := backend.Transport(backend.TransportOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("cannot create transport for tests: %v", err)
|
||||
}
|
||||
|
||||
return &test.Suite{
|
||||
return &test.Suite[rest.Config]{
|
||||
MinimalData: minimalData,
|
||||
|
||||
// NewConfig returns a config for a new temporary backend that will be used in tests.
|
||||
NewConfig: func() (interface{}, error) {
|
||||
NewConfig: func() (*rest.Config, error) {
|
||||
cfg := rest.NewConfig()
|
||||
cfg.URL = url
|
||||
return cfg, nil
|
||||
return &cfg, nil
|
||||
},
|
||||
|
||||
// CreateFn is a function that creates a temporary repository for the tests.
|
||||
Create: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(rest.Config)
|
||||
Create: func(cfg rest.Config) (restic.Backend, error) {
|
||||
return rest.Create(context.TODO(), cfg, tr)
|
||||
},
|
||||
|
||||
// OpenFn is a function that opens a previously created temporary repository.
|
||||
Open: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(rest.Config)
|
||||
Open: func(cfg rest.Config) (restic.Backend, error) {
|
||||
return rest.Open(cfg, tr)
|
||||
},
|
||||
|
||||
// CleanupFn removes data created during the tests.
|
||||
Cleanup: func(config interface{}) error {
|
||||
Cleanup: func(cfg rest.Config) error {
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
@ -130,12 +128,10 @@ func TestBackendRESTExternalServer(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
|
||||
c := cfg.(rest.Config)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
newTestSuite(ctx, t, c.URL, true).RunTests(t)
|
||||
newTestSuite(ctx, t, cfg.URL, true).RunTests(t)
|
||||
}
|
||||
|
||||
func BenchmarkBackendREST(t *testing.B) {
|
||||
|
|
|
@ -2,11 +2,13 @@ package s3
|
|||
|
||||
import (
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/options"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
// Config contains all configuration necessary to connect to an s3 compatible
|
||||
|
@ -44,7 +46,7 @@ func init() {
|
|||
// supported configuration formats are s3://host/bucketname/prefix and
|
||||
// s3:host/bucketname/prefix. The host can also be a valid s3 region
|
||||
// name. If no prefix is given the prefix "restic" will be used.
|
||||
func ParseConfig(s string) (interface{}, error) {
|
||||
func ParseConfig(s string) (*Config, error) {
|
||||
switch {
|
||||
case strings.HasPrefix(s, "s3:http"):
|
||||
// assume that a URL has been specified, parse it and
|
||||
|
@ -75,7 +77,7 @@ func ParseConfig(s string) (interface{}, error) {
|
|||
return createConfig(endpoint, bucket, prefix, false)
|
||||
}
|
||||
|
||||
func createConfig(endpoint, bucket, prefix string, useHTTP bool) (interface{}, error) {
|
||||
func createConfig(endpoint, bucket, prefix string, useHTTP bool) (*Config, error) {
|
||||
if endpoint == "" {
|
||||
return nil, errors.New("s3: invalid format, host/region or bucket name not found")
|
||||
}
|
||||
|
@ -89,5 +91,30 @@ func createConfig(endpoint, bucket, prefix string, useHTTP bool) (interface{}, e
|
|||
cfg.UseHTTP = useHTTP
|
||||
cfg.Bucket = bucket
|
||||
cfg.Prefix = prefix
|
||||
return cfg, nil
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
var _ restic.ApplyEnvironmenter = &Config{}
|
||||
|
||||
// ApplyEnvironment saves values from the environment to the config.
|
||||
func (cfg *Config) ApplyEnvironment(prefix string) error {
|
||||
if cfg.KeyID == "" {
|
||||
cfg.KeyID = os.Getenv(prefix + "AWS_ACCESS_KEY_ID")
|
||||
}
|
||||
|
||||
if cfg.Secret.String() == "" {
|
||||
cfg.Secret = options.NewSecretString(os.Getenv(prefix + "AWS_SECRET_ACCESS_KEY"))
|
||||
}
|
||||
|
||||
if cfg.KeyID == "" && cfg.Secret.String() != "" {
|
||||
return errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty")
|
||||
} else if cfg.KeyID != "" && cfg.Secret.String() == "" {
|
||||
return errors.Fatalf("unable to open S3 backend: Secret ($AWS_SECRET_ACCESS_KEY) is empty")
|
||||
}
|
||||
|
||||
if cfg.Region == "" {
|
||||
cfg.Region = os.Getenv(prefix + "AWS_DEFAULT_REGION")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -3,94 +3,93 @@ package s3
|
|||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/backend/test"
|
||||
)
|
||||
|
||||
var configTests = []struct {
|
||||
s string
|
||||
cfg Config
|
||||
}{
|
||||
{"s3://eu-central-1/bucketname", Config{
|
||||
var configTests = []test.ConfigTestData[Config]{
|
||||
{S: "s3://eu-central-1/bucketname", Cfg: Config{
|
||||
Endpoint: "eu-central-1",
|
||||
Bucket: "bucketname",
|
||||
Prefix: "",
|
||||
Connections: 5,
|
||||
}},
|
||||
{"s3://eu-central-1/bucketname/", Config{
|
||||
{S: "s3://eu-central-1/bucketname/", Cfg: Config{
|
||||
Endpoint: "eu-central-1",
|
||||
Bucket: "bucketname",
|
||||
Prefix: "",
|
||||
Connections: 5,
|
||||
}},
|
||||
{"s3://eu-central-1/bucketname/prefix/directory", Config{
|
||||
{S: "s3://eu-central-1/bucketname/prefix/directory", Cfg: Config{
|
||||
Endpoint: "eu-central-1",
|
||||
Bucket: "bucketname",
|
||||
Prefix: "prefix/directory",
|
||||
Connections: 5,
|
||||
}},
|
||||
{"s3://eu-central-1/bucketname/prefix/directory/", Config{
|
||||
{S: "s3://eu-central-1/bucketname/prefix/directory/", Cfg: Config{
|
||||
Endpoint: "eu-central-1",
|
||||
Bucket: "bucketname",
|
||||
Prefix: "prefix/directory",
|
||||
Connections: 5,
|
||||
}},
|
||||
{"s3:eu-central-1/foobar", Config{
|
||||
{S: "s3:eu-central-1/foobar", Cfg: Config{
|
||||
Endpoint: "eu-central-1",
|
||||
Bucket: "foobar",
|
||||
Prefix: "",
|
||||
Connections: 5,
|
||||
}},
|
||||
{"s3:eu-central-1/foobar/", Config{
|
||||
{S: "s3:eu-central-1/foobar/", Cfg: Config{
|
||||
Endpoint: "eu-central-1",
|
||||
Bucket: "foobar",
|
||||
Prefix: "",
|
||||
Connections: 5,
|
||||
}},
|
||||
{"s3:eu-central-1/foobar/prefix/directory", Config{
|
||||
{S: "s3:eu-central-1/foobar/prefix/directory", Cfg: Config{
|
||||
Endpoint: "eu-central-1",
|
||||
Bucket: "foobar",
|
||||
Prefix: "prefix/directory",
|
||||
Connections: 5,
|
||||
}},
|
||||
{"s3:eu-central-1/foobar/prefix/directory/", Config{
|
||||
{S: "s3:eu-central-1/foobar/prefix/directory/", Cfg: Config{
|
||||
Endpoint: "eu-central-1",
|
||||
Bucket: "foobar",
|
||||
Prefix: "prefix/directory",
|
||||
Connections: 5,
|
||||
}},
|
||||
{"s3:https://hostname:9999/foobar", Config{
|
||||
{S: "s3:https://hostname:9999/foobar", Cfg: Config{
|
||||
Endpoint: "hostname:9999",
|
||||
Bucket: "foobar",
|
||||
Prefix: "",
|
||||
Connections: 5,
|
||||
}},
|
||||
{"s3:https://hostname:9999/foobar/", Config{
|
||||
{S: "s3:https://hostname:9999/foobar/", Cfg: Config{
|
||||
Endpoint: "hostname:9999",
|
||||
Bucket: "foobar",
|
||||
Prefix: "",
|
||||
Connections: 5,
|
||||
}},
|
||||
{"s3:http://hostname:9999/foobar", Config{
|
||||
{S: "s3:http://hostname:9999/foobar", Cfg: Config{
|
||||
Endpoint: "hostname:9999",
|
||||
Bucket: "foobar",
|
||||
Prefix: "",
|
||||
UseHTTP: true,
|
||||
Connections: 5,
|
||||
}},
|
||||
{"s3:http://hostname:9999/foobar/", Config{
|
||||
{S: "s3:http://hostname:9999/foobar/", Cfg: Config{
|
||||
Endpoint: "hostname:9999",
|
||||
Bucket: "foobar",
|
||||
Prefix: "",
|
||||
UseHTTP: true,
|
||||
Connections: 5,
|
||||
}},
|
||||
{"s3:http://hostname:9999/bucket/prefix/directory", Config{
|
||||
{S: "s3:http://hostname:9999/bucket/prefix/directory", Cfg: Config{
|
||||
Endpoint: "hostname:9999",
|
||||
Bucket: "bucket",
|
||||
Prefix: "prefix/directory",
|
||||
UseHTTP: true,
|
||||
Connections: 5,
|
||||
}},
|
||||
{"s3:http://hostname:9999/bucket/prefix/directory/", Config{
|
||||
{S: "s3:http://hostname:9999/bucket/prefix/directory/", Cfg: Config{
|
||||
Endpoint: "hostname:9999",
|
||||
Bucket: "bucket",
|
||||
Prefix: "prefix/directory",
|
||||
|
@ -100,19 +99,7 @@ var configTests = []struct {
|
|||
}
|
||||
|
||||
func TestParseConfig(t *testing.T) {
|
||||
for i, test := range configTests {
|
||||
cfg, err := ParseConfig(test.s)
|
||||
if err != nil {
|
||||
t.Errorf("test %d:%s failed: %v", i, test.s, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if cfg != test.cfg {
|
||||
t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v",
|
||||
i, test.s, test.cfg, cfg)
|
||||
continue
|
||||
}
|
||||
}
|
||||
test.ParseConfigTester(t, ParseConfig, configTests)
|
||||
}
|
||||
|
||||
func TestParseError(t *testing.T) {
|
||||
|
|
|
@ -120,15 +120,15 @@ func createS3(t testing.TB, cfg MinioTestConfig, tr http.RoundTripper) (be resti
|
|||
return be, err
|
||||
}
|
||||
|
||||
func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
|
||||
func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite[MinioTestConfig] {
|
||||
tr, err := backend.Transport(backend.TransportOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("cannot create transport for tests: %v", err)
|
||||
}
|
||||
|
||||
return &test.Suite{
|
||||
return &test.Suite[MinioTestConfig]{
|
||||
// NewConfig returns a config for a new temporary backend that will be used in tests.
|
||||
NewConfig: func() (interface{}, error) {
|
||||
NewConfig: func() (*MinioTestConfig, error) {
|
||||
cfg := MinioTestConfig{}
|
||||
|
||||
cfg.tempdir = rtest.TempDir(t)
|
||||
|
@ -142,13 +142,11 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
|
|||
cfg.Config.UseHTTP = true
|
||||
cfg.Config.KeyID = key
|
||||
cfg.Config.Secret = options.NewSecretString(secret)
|
||||
return cfg, nil
|
||||
return &cfg, nil
|
||||
},
|
||||
|
||||
// CreateFn is a function that creates a temporary repository for the tests.
|
||||
Create: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(MinioTestConfig)
|
||||
|
||||
Create: func(cfg MinioTestConfig) (restic.Backend, error) {
|
||||
be, err := createS3(t, cfg, tr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
@ -167,14 +165,12 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
|
|||
},
|
||||
|
||||
// OpenFn is a function that opens a previously created temporary repository.
|
||||
Open: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(MinioTestConfig)
|
||||
Open: func(cfg MinioTestConfig) (restic.Backend, error) {
|
||||
return s3.Open(ctx, cfg.Config, tr)
|
||||
},
|
||||
|
||||
// CleanupFn removes data created during the tests.
|
||||
Cleanup: func(config interface{}) error {
|
||||
cfg := config.(MinioTestConfig)
|
||||
Cleanup: func(cfg MinioTestConfig) error {
|
||||
if cfg.stopServer != nil {
|
||||
cfg.stopServer()
|
||||
}
|
||||
|
@ -217,24 +213,23 @@ func BenchmarkBackendMinio(t *testing.B) {
|
|||
newMinioTestSuite(ctx, t).RunBenchmarks(t)
|
||||
}
|
||||
|
||||
func newS3TestSuite(t testing.TB) *test.Suite {
|
||||
func newS3TestSuite(t testing.TB) *test.Suite[s3.Config] {
|
||||
tr, err := backend.Transport(backend.TransportOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("cannot create transport for tests: %v", err)
|
||||
}
|
||||
|
||||
return &test.Suite{
|
||||
return &test.Suite[s3.Config]{
|
||||
// do not use excessive data
|
||||
MinimalData: true,
|
||||
|
||||
// NewConfig returns a config for a new temporary backend that will be used in tests.
|
||||
NewConfig: func() (interface{}, error) {
|
||||
s3cfg, err := s3.ParseConfig(os.Getenv("RESTIC_TEST_S3_REPOSITORY"))
|
||||
NewConfig: func() (*s3.Config, error) {
|
||||
cfg, err := s3.ParseConfig(os.Getenv("RESTIC_TEST_S3_REPOSITORY"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg := s3cfg.(s3.Config)
|
||||
cfg.KeyID = os.Getenv("RESTIC_TEST_S3_KEY")
|
||||
cfg.Secret = options.NewSecretString(os.Getenv("RESTIC_TEST_S3_SECRET"))
|
||||
cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
|
||||
|
@ -242,9 +237,7 @@ func newS3TestSuite(t testing.TB) *test.Suite {
|
|||
},
|
||||
|
||||
// CreateFn is a function that creates a temporary repository for the tests.
|
||||
Create: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(s3.Config)
|
||||
|
||||
Create: func(cfg s3.Config) (restic.Backend, error) {
|
||||
be, err := s3.Create(context.TODO(), cfg, tr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
@ -263,15 +256,12 @@ func newS3TestSuite(t testing.TB) *test.Suite {
|
|||
},
|
||||
|
||||
// OpenFn is a function that opens a previously created temporary repository.
|
||||
Open: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(s3.Config)
|
||||
Open: func(cfg s3.Config) (restic.Backend, error) {
|
||||
return s3.Open(context.TODO(), cfg, tr)
|
||||
},
|
||||
|
||||
// CleanupFn removes data created during the tests.
|
||||
Cleanup: func(config interface{}) error {
|
||||
cfg := config.(s3.Config)
|
||||
|
||||
Cleanup: func(cfg s3.Config) error {
|
||||
be, err := s3.Open(context.TODO(), cfg, tr)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
@ -35,7 +35,7 @@ func init() {
|
|||
// and sftp:user@host:directory. The directory will be path Cleaned and can
|
||||
// be an absolute path if it starts with a '/' (e.g.
|
||||
// sftp://user@host//absolute and sftp:user@host:/absolute).
|
||||
func ParseConfig(s string) (interface{}, error) {
|
||||
func ParseConfig(s string) (*Config, error) {
|
||||
var user, host, port, dir string
|
||||
switch {
|
||||
case strings.HasPrefix(s, "sftp://"):
|
||||
|
@ -89,5 +89,5 @@ func ParseConfig(s string) (interface{}, error) {
|
|||
cfg.Port = port
|
||||
cfg.Path = p
|
||||
|
||||
return cfg, nil
|
||||
return &cfg, nil
|
||||
}
|
||||
|
|
|
@ -2,94 +2,81 @@ package sftp
|
|||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/backend/test"
|
||||
)
|
||||
|
||||
var configTests = []struct {
|
||||
in string
|
||||
cfg Config
|
||||
}{
|
||||
var configTests = []test.ConfigTestData[Config]{
|
||||
// first form, user specified sftp://user@host/dir
|
||||
{
|
||||
"sftp://user@host/dir/subdir",
|
||||
Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5},
|
||||
S: "sftp://user@host/dir/subdir",
|
||||
Cfg: Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5},
|
||||
},
|
||||
{
|
||||
"sftp://host/dir/subdir",
|
||||
Config{Host: "host", Path: "dir/subdir", Connections: 5},
|
||||
S: "sftp://host/dir/subdir",
|
||||
Cfg: Config{Host: "host", Path: "dir/subdir", Connections: 5},
|
||||
},
|
||||
{
|
||||
"sftp://host//dir/subdir",
|
||||
Config{Host: "host", Path: "/dir/subdir", Connections: 5},
|
||||
S: "sftp://host//dir/subdir",
|
||||
Cfg: Config{Host: "host", Path: "/dir/subdir", Connections: 5},
|
||||
},
|
||||
{
|
||||
"sftp://host:10022//dir/subdir",
|
||||
Config{Host: "host", Port: "10022", Path: "/dir/subdir", Connections: 5},
|
||||
S: "sftp://host:10022//dir/subdir",
|
||||
Cfg: Config{Host: "host", Port: "10022", Path: "/dir/subdir", Connections: 5},
|
||||
},
|
||||
{
|
||||
"sftp://user@host:10022//dir/subdir",
|
||||
Config{User: "user", Host: "host", Port: "10022", Path: "/dir/subdir", Connections: 5},
|
||||
S: "sftp://user@host:10022//dir/subdir",
|
||||
Cfg: Config{User: "user", Host: "host", Port: "10022", Path: "/dir/subdir", Connections: 5},
|
||||
},
|
||||
{
|
||||
"sftp://user@host/dir/subdir/../other",
|
||||
Config{User: "user", Host: "host", Path: "dir/other", Connections: 5},
|
||||
S: "sftp://user@host/dir/subdir/../other",
|
||||
Cfg: Config{User: "user", Host: "host", Path: "dir/other", Connections: 5},
|
||||
},
|
||||
{
|
||||
"sftp://user@host/dir///subdir",
|
||||
Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5},
|
||||
S: "sftp://user@host/dir///subdir",
|
||||
Cfg: Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5},
|
||||
},
|
||||
|
||||
// IPv6 address.
|
||||
{
|
||||
"sftp://user@[::1]/dir",
|
||||
Config{User: "user", Host: "::1", Path: "dir", Connections: 5},
|
||||
S: "sftp://user@[::1]/dir",
|
||||
Cfg: Config{User: "user", Host: "::1", Path: "dir", Connections: 5},
|
||||
},
|
||||
// IPv6 address with port.
|
||||
{
|
||||
"sftp://user@[::1]:22/dir",
|
||||
Config{User: "user", Host: "::1", Port: "22", Path: "dir", Connections: 5},
|
||||
S: "sftp://user@[::1]:22/dir",
|
||||
Cfg: Config{User: "user", Host: "::1", Port: "22", Path: "dir", Connections: 5},
|
||||
},
|
||||
|
||||
// second form, user specified sftp:user@host:/dir
|
||||
{
|
||||
"sftp:user@host:/dir/subdir",
|
||||
Config{User: "user", Host: "host", Path: "/dir/subdir", Connections: 5},
|
||||
S: "sftp:user@host:/dir/subdir",
|
||||
Cfg: Config{User: "user", Host: "host", Path: "/dir/subdir", Connections: 5},
|
||||
},
|
||||
{
|
||||
"sftp:user@domain@host:/dir/subdir",
|
||||
Config{User: "user@domain", Host: "host", Path: "/dir/subdir", Connections: 5},
|
||||
S: "sftp:user@domain@host:/dir/subdir",
|
||||
Cfg: Config{User: "user@domain", Host: "host", Path: "/dir/subdir", Connections: 5},
|
||||
},
|
||||
{
|
||||
"sftp:host:../dir/subdir",
|
||||
Config{Host: "host", Path: "../dir/subdir", Connections: 5},
|
||||
S: "sftp:host:../dir/subdir",
|
||||
Cfg: Config{Host: "host", Path: "../dir/subdir", Connections: 5},
|
||||
},
|
||||
{
|
||||
"sftp:user@host:dir/subdir:suffix",
|
||||
Config{User: "user", Host: "host", Path: "dir/subdir:suffix", Connections: 5},
|
||||
S: "sftp:user@host:dir/subdir:suffix",
|
||||
Cfg: Config{User: "user", Host: "host", Path: "dir/subdir:suffix", Connections: 5},
|
||||
},
|
||||
{
|
||||
"sftp:user@host:dir/subdir/../other",
|
||||
Config{User: "user", Host: "host", Path: "dir/other", Connections: 5},
|
||||
S: "sftp:user@host:dir/subdir/../other",
|
||||
Cfg: Config{User: "user", Host: "host", Path: "dir/other", Connections: 5},
|
||||
},
|
||||
{
|
||||
"sftp:user@host:dir///subdir",
|
||||
Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5},
|
||||
S: "sftp:user@host:dir///subdir",
|
||||
Cfg: Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5},
|
||||
},
|
||||
}
|
||||
|
||||
func TestParseConfig(t *testing.T) {
|
||||
for i, test := range configTests {
|
||||
cfg, err := ParseConfig(test.in)
|
||||
if err != nil {
|
||||
t.Errorf("test %d:%s failed: %v", i, test.in, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if cfg != test.cfg {
|
||||
t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v",
|
||||
i, test.in, test.cfg, cfg)
|
||||
continue
|
||||
}
|
||||
}
|
||||
test.ParseConfigTester(t, ParseConfig, configTests)
|
||||
}
|
||||
|
||||
var configTestsInvalid = []string{
|
||||
|
|
|
@ -29,10 +29,10 @@ func findSFTPServerBinary() string {
|
|||
|
||||
var sftpServer = findSFTPServerBinary()
|
||||
|
||||
func newTestSuite(t testing.TB) *test.Suite {
|
||||
return &test.Suite{
|
||||
func newTestSuite(t testing.TB) *test.Suite[sftp.Config] {
|
||||
return &test.Suite[sftp.Config]{
|
||||
// NewConfig returns a config for a new temporary backend that will be used in tests.
|
||||
NewConfig: func() (interface{}, error) {
|
||||
NewConfig: func() (*sftp.Config, error) {
|
||||
dir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-sftp-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
@ -40,7 +40,7 @@ func newTestSuite(t testing.TB) *test.Suite {
|
|||
|
||||
t.Logf("create new backend at %v", dir)
|
||||
|
||||
cfg := sftp.Config{
|
||||
cfg := &sftp.Config{
|
||||
Path: dir,
|
||||
Command: fmt.Sprintf("%q -e", sftpServer),
|
||||
Connections: 5,
|
||||
|
@ -49,20 +49,17 @@ func newTestSuite(t testing.TB) *test.Suite {
|
|||
},
|
||||
|
||||
// CreateFn is a function that creates a temporary repository for the tests.
|
||||
Create: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(sftp.Config)
|
||||
Create: func(cfg sftp.Config) (restic.Backend, error) {
|
||||
return sftp.Create(context.TODO(), cfg)
|
||||
},
|
||||
|
||||
// OpenFn is a function that opens a previously created temporary repository.
|
||||
Open: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(sftp.Config)
|
||||
Open: func(cfg sftp.Config) (restic.Backend, error) {
|
||||
return sftp.Open(context.TODO(), cfg)
|
||||
},
|
||||
|
||||
// CleanupFn removes data created during the tests.
|
||||
Cleanup: func(config interface{}) error {
|
||||
cfg := config.(sftp.Config)
|
||||
Cleanup: func(cfg sftp.Config) error {
|
||||
if !rtest.TestCleanupTempDirs {
|
||||
t.Logf("leaving test backend dir at %v", cfg.Path)
|
||||
}
|
||||
|
|
|
@ -2,6 +2,7 @@ package smb
|
|||
|
||||
import (
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
@ -9,6 +10,7 @@ import (
|
|||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/options"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
// Config contains all configuration necessary to connect to an SMB server
|
||||
|
@ -54,7 +56,7 @@ func init() {
|
|||
// ParseConfig parses the string s and extracts the s3 config. The
|
||||
// supported configuration format is smb://[user@]host[:port]/sharename/directory.
|
||||
// User and port are optional. Default port is 445.
|
||||
func ParseConfig(s string) (interface{}, error) {
|
||||
func ParseConfig(s string) (*Config, error) {
|
||||
var repo string
|
||||
switch {
|
||||
case strings.HasPrefix(s, "smb://"):
|
||||
|
@ -106,7 +108,7 @@ func ParseConfig(s string) (interface{}, error) {
|
|||
return createConfig(user, host, portNum, sharename, directory)
|
||||
}
|
||||
|
||||
func createConfig(user string, host string, port int, sharename, directory string) (interface{}, error) {
|
||||
func createConfig(user string, host string, port int, sharename, directory string) (*Config, error) {
|
||||
if host == "" {
|
||||
return nil, errors.New("smb: invalid format, Host not found")
|
||||
}
|
||||
|
@ -121,5 +123,26 @@ func createConfig(user string, host string, port int, sharename, directory strin
|
|||
cfg.Port = port
|
||||
cfg.ShareName = sharename
|
||||
cfg.Path = directory
|
||||
return cfg, nil
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
var _ restic.ApplyEnvironmenter = &Config{}
|
||||
|
||||
// ApplyEnvironment saves values from the environment to the config.
|
||||
func (cfg *Config) ApplyEnvironment(prefix string) error {
|
||||
if cfg.User == "" {
|
||||
cfg.User = os.Getenv("RESTIC_SMB_USER")
|
||||
}
|
||||
|
||||
if cfg.Password.String() == "" {
|
||||
cfg.Password = options.NewSecretString(os.Getenv("RESTIC_SMB_PASSWORD"))
|
||||
}
|
||||
|
||||
if cfg.Domain == "" {
|
||||
cfg.Domain = os.Getenv("RESTIC_SMB_DOMAIN")
|
||||
}
|
||||
if cfg.Domain == "" {
|
||||
cfg.Domain = DefaultDomain
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -3,13 +3,13 @@ package smb
|
|||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/backend/test"
|
||||
)
|
||||
|
||||
var configTests = []struct {
|
||||
s string
|
||||
cfg Config
|
||||
}{
|
||||
{"smb://user@host/sharename/directory", Config{
|
||||
var configTests = []test.ConfigTestData[Config]{
|
||||
{S: "smb://user@host/sharename/directory",
|
||||
Cfg: Config{
|
||||
Host: "host",
|
||||
Port: DefaultSmbPort,
|
||||
User: "user",
|
||||
|
@ -19,7 +19,8 @@ var configTests = []struct {
|
|||
Connections: DefaultConnections,
|
||||
IdleTimeout: DefaultIdleTimeout,
|
||||
}},
|
||||
{"smb://user@host:456/sharename/directory", Config{
|
||||
{S: "smb://user@host:456/sharename/directory",
|
||||
Cfg: Config{
|
||||
Host: "host",
|
||||
Port: 456,
|
||||
User: "user",
|
||||
|
@ -29,7 +30,8 @@ var configTests = []struct {
|
|||
Connections: DefaultConnections,
|
||||
IdleTimeout: DefaultIdleTimeout,
|
||||
}},
|
||||
{"smb://host/sharename/directory", Config{
|
||||
{S: "smb://host/sharename/directory",
|
||||
Cfg: Config{
|
||||
Host: "host",
|
||||
Port: DefaultSmbPort,
|
||||
Domain: DefaultDomain,
|
||||
|
@ -38,7 +40,8 @@ var configTests = []struct {
|
|||
Connections: DefaultConnections,
|
||||
IdleTimeout: DefaultIdleTimeout,
|
||||
}},
|
||||
{"smb://host:446/sharename/directory", Config{
|
||||
{S: "smb://host:446/sharename/directory",
|
||||
Cfg: Config{
|
||||
Host: "host",
|
||||
Port: 446,
|
||||
Domain: DefaultDomain,
|
||||
|
@ -47,7 +50,8 @@ var configTests = []struct {
|
|||
Connections: DefaultConnections,
|
||||
IdleTimeout: DefaultIdleTimeout,
|
||||
}},
|
||||
{"smb:user@host:466/sharename/directory", Config{
|
||||
{S: "smb:user@host:466/sharename/directory",
|
||||
Cfg: Config{
|
||||
Host: "host",
|
||||
Port: 466,
|
||||
User: "user",
|
||||
|
@ -60,19 +64,7 @@ var configTests = []struct {
|
|||
}
|
||||
|
||||
func TestParseConfig(t *testing.T) {
|
||||
for i, test := range configTests {
|
||||
cfg, err := ParseConfig(test.s)
|
||||
if err != nil {
|
||||
t.Errorf("test %d:%s failed: %v", i, test.s, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if cfg != test.cfg {
|
||||
t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v",
|
||||
i, test.s, test.cfg, cfg)
|
||||
continue
|
||||
}
|
||||
}
|
||||
test.ParseConfigTester(t, ParseConfig, configTests)
|
||||
}
|
||||
|
||||
func TestParseError(t *testing.T) {
|
||||
|
|
|
@ -13,12 +13,12 @@ import (
|
|||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
func newTestSuite(t testing.TB) *test.Suite {
|
||||
return &test.Suite{
|
||||
func newTestSuite(t testing.TB) *test.Suite[smb.Config] {
|
||||
return &test.Suite[smb.Config]{
|
||||
// NewConfig returns a config for a new temporary backend that will be used in tests.
|
||||
NewConfig: func() (interface{}, error) {
|
||||
NewConfig: func() (*smb.Config, error) {
|
||||
|
||||
cfg := smb.NewConfig()
|
||||
cfg := &smb.Config{}
|
||||
cfg.Host = "127.0.0.1"
|
||||
cfg.User = "smbuser"
|
||||
cfg.ShareName = cfg.User
|
||||
|
@ -38,20 +38,17 @@ func newTestSuite(t testing.TB) *test.Suite {
|
|||
},
|
||||
|
||||
// CreateFn is a function that creates a temporary repository for the tests.
|
||||
Create: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(smb.Config)
|
||||
Create: func(cfg smb.Config) (restic.Backend, error) {
|
||||
return smb.Create(context.TODO(), cfg)
|
||||
},
|
||||
|
||||
// OpenFn is a function that opens a previously created temporary repository.
|
||||
Open: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(smb.Config)
|
||||
Open: func(cfg smb.Config) (restic.Backend, error) {
|
||||
return smb.Open(context.TODO(), cfg)
|
||||
},
|
||||
|
||||
// CleanupFn removes data created during the tests.
|
||||
Cleanup: func(config interface{}) error {
|
||||
cfg := config.(smb.Config)
|
||||
Cleanup: func(cfg smb.Config) error {
|
||||
if !rtest.TestCleanupTempDirs {
|
||||
t.Logf("leaving test backend dir at %v", cfg.Path)
|
||||
}
|
||||
|
|
|
@ -6,6 +6,7 @@ import (
|
|||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/options"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
// Config contains basic configuration needed to specify swift location for a swift server
|
||||
|
@ -50,7 +51,7 @@ func NewConfig() Config {
|
|||
}
|
||||
|
||||
// ParseConfig parses the string s and extract swift's container name and prefix.
|
||||
func ParseConfig(s string) (interface{}, error) {
|
||||
func ParseConfig(s string) (*Config, error) {
|
||||
if !strings.HasPrefix(s, "swift:") {
|
||||
return nil, errors.New("invalid URL, expected: swift:container-name:/[prefix]")
|
||||
}
|
||||
|
@ -70,48 +71,49 @@ func ParseConfig(s string) (interface{}, error) {
|
|||
cfg.Container = container
|
||||
cfg.Prefix = prefix
|
||||
|
||||
return cfg, nil
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
var _ restic.ApplyEnvironmenter = &Config{}
|
||||
|
||||
// ApplyEnvironment saves values from the environment to the config.
|
||||
func ApplyEnvironment(prefix string, cfg interface{}) error {
|
||||
c := cfg.(*Config)
|
||||
func (cfg *Config) ApplyEnvironment(prefix string) error {
|
||||
for _, val := range []struct {
|
||||
s *string
|
||||
env string
|
||||
}{
|
||||
// v2/v3 specific
|
||||
{&c.UserName, prefix + "OS_USERNAME"},
|
||||
{&c.APIKey, prefix + "OS_PASSWORD"},
|
||||
{&c.Region, prefix + "OS_REGION_NAME"},
|
||||
{&c.AuthURL, prefix + "OS_AUTH_URL"},
|
||||
{&cfg.UserName, prefix + "OS_USERNAME"},
|
||||
{&cfg.APIKey, prefix + "OS_PASSWORD"},
|
||||
{&cfg.Region, prefix + "OS_REGION_NAME"},
|
||||
{&cfg.AuthURL, prefix + "OS_AUTH_URL"},
|
||||
|
||||
// v3 specific
|
||||
{&c.UserID, prefix + "OS_USER_ID"},
|
||||
{&c.Domain, prefix + "OS_USER_DOMAIN_NAME"},
|
||||
{&c.DomainID, prefix + "OS_USER_DOMAIN_ID"},
|
||||
{&c.Tenant, prefix + "OS_PROJECT_NAME"},
|
||||
{&c.TenantDomain, prefix + "OS_PROJECT_DOMAIN_NAME"},
|
||||
{&c.TenantDomainID, prefix + "OS_PROJECT_DOMAIN_ID"},
|
||||
{&c.TrustID, prefix + "OS_TRUST_ID"},
|
||||
{&cfg.UserID, prefix + "OS_USER_ID"},
|
||||
{&cfg.Domain, prefix + "OS_USER_DOMAIN_NAME"},
|
||||
{&cfg.DomainID, prefix + "OS_USER_DOMAIN_ID"},
|
||||
{&cfg.Tenant, prefix + "OS_PROJECT_NAME"},
|
||||
{&cfg.TenantDomain, prefix + "OS_PROJECT_DOMAIN_NAME"},
|
||||
{&cfg.TenantDomainID, prefix + "OS_PROJECT_DOMAIN_ID"},
|
||||
{&cfg.TrustID, prefix + "OS_TRUST_ID"},
|
||||
|
||||
// v2 specific
|
||||
{&c.TenantID, prefix + "OS_TENANT_ID"},
|
||||
{&c.Tenant, prefix + "OS_TENANT_NAME"},
|
||||
{&cfg.TenantID, prefix + "OS_TENANT_ID"},
|
||||
{&cfg.Tenant, prefix + "OS_TENANT_NAME"},
|
||||
|
||||
// v1 specific
|
||||
{&c.AuthURL, prefix + "ST_AUTH"},
|
||||
{&c.UserName, prefix + "ST_USER"},
|
||||
{&c.APIKey, prefix + "ST_KEY"},
|
||||
{&cfg.AuthURL, prefix + "ST_AUTH"},
|
||||
{&cfg.UserName, prefix + "ST_USER"},
|
||||
{&cfg.APIKey, prefix + "ST_KEY"},
|
||||
|
||||
// Application Credential auth
|
||||
{&c.ApplicationCredentialID, prefix + "OS_APPLICATION_CREDENTIAL_ID"},
|
||||
{&c.ApplicationCredentialName, prefix + "OS_APPLICATION_CREDENTIAL_NAME"},
|
||||
{&cfg.ApplicationCredentialID, prefix + "OS_APPLICATION_CREDENTIAL_ID"},
|
||||
{&cfg.ApplicationCredentialName, prefix + "OS_APPLICATION_CREDENTIAL_NAME"},
|
||||
|
||||
// Manual authentication
|
||||
{&c.StorageURL, prefix + "OS_STORAGE_URL"},
|
||||
{&cfg.StorageURL, prefix + "OS_STORAGE_URL"},
|
||||
|
||||
{&c.DefaultContainerPolicy, prefix + "SWIFT_DEFAULT_CONTAINER_POLICY"},
|
||||
{&cfg.DefaultContainerPolicy, prefix + "SWIFT_DEFAULT_CONTAINER_POLICY"},
|
||||
} {
|
||||
if *val.s == "" {
|
||||
*val.s = os.Getenv(val.env)
|
||||
|
@ -121,8 +123,8 @@ func ApplyEnvironment(prefix string, cfg interface{}) error {
|
|||
s *options.SecretString
|
||||
env string
|
||||
}{
|
||||
{&c.ApplicationCredentialSecret, prefix + "OS_APPLICATION_CREDENTIAL_SECRET"},
|
||||
{&c.AuthToken, prefix + "OS_AUTH_TOKEN"},
|
||||
{&cfg.ApplicationCredentialSecret, prefix + "OS_APPLICATION_CREDENTIAL_SECRET"},
|
||||
{&cfg.AuthToken, prefix + "OS_AUTH_TOKEN"},
|
||||
} {
|
||||
if val.s.String() == "" {
|
||||
*val.s = options.NewSecretString(os.Getenv(val.env))
|
||||
|
|
|
@ -1,29 +1,30 @@
|
|||
package swift
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"testing"
|
||||
|
||||
var configTests = []struct {
|
||||
s string
|
||||
cfg Config
|
||||
}{
|
||||
"github.com/restic/restic/internal/backend/test"
|
||||
)
|
||||
|
||||
var configTests = []test.ConfigTestData[Config]{
|
||||
{
|
||||
"swift:cnt1:/",
|
||||
Config{
|
||||
S: "swift:cnt1:/",
|
||||
Cfg: Config{
|
||||
Container: "cnt1",
|
||||
Prefix: "",
|
||||
Connections: 5,
|
||||
},
|
||||
},
|
||||
{
|
||||
"swift:cnt2:/prefix",
|
||||
Config{Container: "cnt2",
|
||||
S: "swift:cnt2:/prefix",
|
||||
Cfg: Config{Container: "cnt2",
|
||||
Prefix: "prefix",
|
||||
Connections: 5,
|
||||
},
|
||||
},
|
||||
{
|
||||
"swift:cnt3:/prefix/longer",
|
||||
Config{Container: "cnt3",
|
||||
S: "swift:cnt3:/prefix/longer",
|
||||
Cfg: Config{Container: "cnt3",
|
||||
Prefix: "prefix/longer",
|
||||
Connections: 5,
|
||||
},
|
||||
|
@ -31,24 +32,7 @@ var configTests = []struct {
|
|||
}
|
||||
|
||||
func TestParseConfig(t *testing.T) {
|
||||
for _, test := range configTests {
|
||||
t.Run("", func(t *testing.T) {
|
||||
v, err := ParseConfig(test.s)
|
||||
if err != nil {
|
||||
t.Fatalf("parsing %q failed: %v", test.s, err)
|
||||
}
|
||||
|
||||
cfg, ok := v.(Config)
|
||||
if !ok {
|
||||
t.Fatalf("wrong type returned, want Config, got %T", cfg)
|
||||
}
|
||||
|
||||
if cfg != test.cfg {
|
||||
t.Fatalf("wrong output for %q, want:\n %#v\ngot:\n %#v",
|
||||
test.s, test.cfg, cfg)
|
||||
}
|
||||
})
|
||||
}
|
||||
test.ParseConfigTester(t, ParseConfig, configTests)
|
||||
}
|
||||
|
||||
var configTestsInvalid = []string{
|
||||
|
|
|
@ -15,13 +15,13 @@ import (
|
|||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
func newSwiftTestSuite(t testing.TB) *test.Suite {
|
||||
func newSwiftTestSuite(t testing.TB) *test.Suite[swift.Config] {
|
||||
tr, err := backend.Transport(backend.TransportOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("cannot create transport for tests: %v", err)
|
||||
}
|
||||
|
||||
return &test.Suite{
|
||||
return &test.Suite[swift.Config]{
|
||||
// do not use excessive data
|
||||
MinimalData: true,
|
||||
|
||||
|
@ -42,14 +42,13 @@ func newSwiftTestSuite(t testing.TB) *test.Suite {
|
|||
},
|
||||
|
||||
// NewConfig returns a config for a new temporary backend that will be used in tests.
|
||||
NewConfig: func() (interface{}, error) {
|
||||
swiftcfg, err := swift.ParseConfig(os.Getenv("RESTIC_TEST_SWIFT"))
|
||||
NewConfig: func() (*swift.Config, error) {
|
||||
cfg, err := swift.ParseConfig(os.Getenv("RESTIC_TEST_SWIFT"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg := swiftcfg.(swift.Config)
|
||||
if err = swift.ApplyEnvironment("RESTIC_TEST_", &cfg); err != nil {
|
||||
if err = cfg.ApplyEnvironment("RESTIC_TEST_"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.Prefix += fmt.Sprintf("/test-%d", time.Now().UnixNano())
|
||||
|
@ -58,9 +57,7 @@ func newSwiftTestSuite(t testing.TB) *test.Suite {
|
|||
},
|
||||
|
||||
// CreateFn is a function that creates a temporary repository for the tests.
|
||||
Create: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(swift.Config)
|
||||
|
||||
Create: func(cfg swift.Config) (restic.Backend, error) {
|
||||
be, err := swift.Open(context.TODO(), cfg, tr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
@ -79,15 +76,12 @@ func newSwiftTestSuite(t testing.TB) *test.Suite {
|
|||
},
|
||||
|
||||
// OpenFn is a function that opens a previously created temporary repository.
|
||||
Open: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(swift.Config)
|
||||
Open: func(cfg swift.Config) (restic.Backend, error) {
|
||||
return swift.Open(context.TODO(), cfg, tr)
|
||||
},
|
||||
|
||||
// CleanupFn removes data created during the tests.
|
||||
Cleanup: func(config interface{}) error {
|
||||
cfg := config.(swift.Config)
|
||||
|
||||
Cleanup: func(cfg swift.Config) error {
|
||||
be, err := swift.Open(context.TODO(), cfg, tr)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
@ -29,7 +29,7 @@ func remove(t testing.TB, be restic.Backend, h restic.Handle) {
|
|||
|
||||
// BenchmarkLoadFile benchmarks the Load() method of a backend by
|
||||
// loading a complete file.
|
||||
func (s *Suite) BenchmarkLoadFile(t *testing.B) {
|
||||
func (s *Suite[C]) BenchmarkLoadFile(t *testing.B) {
|
||||
be := s.open(t)
|
||||
defer s.close(t, be)
|
||||
|
||||
|
@ -64,7 +64,7 @@ func (s *Suite) BenchmarkLoadFile(t *testing.B) {
|
|||
|
||||
// BenchmarkLoadPartialFile benchmarks the Load() method of a backend by
|
||||
// loading the remainder of a file starting at a given offset.
|
||||
func (s *Suite) BenchmarkLoadPartialFile(t *testing.B) {
|
||||
func (s *Suite[C]) BenchmarkLoadPartialFile(t *testing.B) {
|
||||
be := s.open(t)
|
||||
defer s.close(t, be)
|
||||
|
||||
|
@ -101,7 +101,7 @@ func (s *Suite) BenchmarkLoadPartialFile(t *testing.B) {
|
|||
|
||||
// BenchmarkLoadPartialFileOffset benchmarks the Load() method of a
|
||||
// backend by loading a number of bytes of a file starting at a given offset.
|
||||
func (s *Suite) BenchmarkLoadPartialFileOffset(t *testing.B) {
|
||||
func (s *Suite[C]) BenchmarkLoadPartialFileOffset(t *testing.B) {
|
||||
be := s.open(t)
|
||||
defer s.close(t, be)
|
||||
|
||||
|
@ -139,7 +139,7 @@ func (s *Suite) BenchmarkLoadPartialFileOffset(t *testing.B) {
|
|||
}
|
||||
|
||||
// BenchmarkSave benchmarks the Save() method of a backend.
|
||||
func (s *Suite) BenchmarkSave(t *testing.B) {
|
||||
func (s *Suite[C]) BenchmarkSave(t *testing.B) {
|
||||
be := s.open(t)
|
||||
defer s.close(t, be)
|
||||
|
||||
|
|
28
internal/backend/test/config.go
Normal file
28
internal/backend/test/config.go
Normal file
|
@ -0,0 +1,28 @@
|
|||
package test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type ConfigTestData[C comparable] struct {
|
||||
S string
|
||||
Cfg C
|
||||
}
|
||||
|
||||
func ParseConfigTester[C comparable](t *testing.T, parser func(s string) (*C, error), tests []ConfigTestData[C]) {
|
||||
for i, test := range tests {
|
||||
t.Run(fmt.Sprint(i), func(t *testing.T) {
|
||||
cfg, err := parser(test.S)
|
||||
if err != nil {
|
||||
t.Fatalf("%s failed: %v", test.S, err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(*cfg, test.Cfg) {
|
||||
t.Fatalf("input: %s\n wrong config, want:\n %#v\ngot:\n %#v",
|
||||
test.S, test.Cfg, *cfg)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
|
@ -11,21 +11,21 @@ import (
|
|||
)
|
||||
|
||||
// Suite implements a test suite for restic backends.
|
||||
type Suite struct {
|
||||
type Suite[C any] struct {
|
||||
// Config should be used to configure the backend.
|
||||
Config interface{}
|
||||
Config *C
|
||||
|
||||
// NewConfig returns a config for a new temporary backend that will be used in tests.
|
||||
NewConfig func() (interface{}, error)
|
||||
NewConfig func() (*C, error)
|
||||
|
||||
// CreateFn is a function that creates a temporary repository for the tests.
|
||||
Create func(cfg interface{}) (restic.Backend, error)
|
||||
Create func(cfg C) (restic.Backend, error)
|
||||
|
||||
// OpenFn is a function that opens a previously created temporary repository.
|
||||
Open func(cfg interface{}) (restic.Backend, error)
|
||||
Open func(cfg C) (restic.Backend, error)
|
||||
|
||||
// CleanupFn removes data created during the tests.
|
||||
Cleanup func(cfg interface{}) error
|
||||
Cleanup func(cfg C) error
|
||||
|
||||
// MinimalData instructs the tests to not use excessive data.
|
||||
MinimalData bool
|
||||
|
@ -40,7 +40,7 @@ type Suite struct {
|
|||
}
|
||||
|
||||
// RunTests executes all defined tests as subtests of t.
|
||||
func (s *Suite) RunTests(t *testing.T) {
|
||||
func (s *Suite[C]) RunTests(t *testing.T) {
|
||||
var err error
|
||||
s.Config, err = s.NewConfig()
|
||||
if err != nil {
|
||||
|
@ -61,7 +61,7 @@ func (s *Suite) RunTests(t *testing.T) {
|
|||
}
|
||||
|
||||
if s.Cleanup != nil {
|
||||
if err = s.Cleanup(s.Config); err != nil {
|
||||
if err = s.Cleanup(*s.Config); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
@ -72,7 +72,7 @@ type testFunction struct {
|
|||
Fn func(*testing.T)
|
||||
}
|
||||
|
||||
func (s *Suite) testFuncs(t testing.TB) (funcs []testFunction) {
|
||||
func (s *Suite[C]) testFuncs(t testing.TB) (funcs []testFunction) {
|
||||
tpe := reflect.TypeOf(s)
|
||||
v := reflect.ValueOf(s)
|
||||
|
||||
|
@ -107,7 +107,7 @@ type benchmarkFunction struct {
|
|||
Fn func(*testing.B)
|
||||
}
|
||||
|
||||
func (s *Suite) benchmarkFuncs(t testing.TB) (funcs []benchmarkFunction) {
|
||||
func (s *Suite[C]) benchmarkFuncs(t testing.TB) (funcs []benchmarkFunction) {
|
||||
tpe := reflect.TypeOf(s)
|
||||
v := reflect.ValueOf(s)
|
||||
|
||||
|
@ -138,7 +138,7 @@ func (s *Suite) benchmarkFuncs(t testing.TB) (funcs []benchmarkFunction) {
|
|||
}
|
||||
|
||||
// RunBenchmarks executes all defined benchmarks as subtests of b.
|
||||
func (s *Suite) RunBenchmarks(b *testing.B) {
|
||||
func (s *Suite[C]) RunBenchmarks(b *testing.B) {
|
||||
var err error
|
||||
s.Config, err = s.NewConfig()
|
||||
if err != nil {
|
||||
|
@ -158,28 +158,28 @@ func (s *Suite) RunBenchmarks(b *testing.B) {
|
|||
return
|
||||
}
|
||||
|
||||
if err = s.Cleanup(s.Config); err != nil {
|
||||
if err = s.Cleanup(*s.Config); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Suite) create(t testing.TB) restic.Backend {
|
||||
be, err := s.Create(s.Config)
|
||||
func (s *Suite[C]) create(t testing.TB) restic.Backend {
|
||||
be, err := s.Create(*s.Config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return be
|
||||
}
|
||||
|
||||
func (s *Suite) open(t testing.TB) restic.Backend {
|
||||
be, err := s.Open(s.Config)
|
||||
func (s *Suite[C]) open(t testing.TB) restic.Backend {
|
||||
be, err := s.Open(*s.Config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return be
|
||||
}
|
||||
|
||||
func (s *Suite) close(t testing.TB, be restic.Backend) {
|
||||
func (s *Suite[C]) close(t testing.TB, be restic.Backend) {
|
||||
err := be.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
|
|
@ -38,7 +38,7 @@ func beTest(ctx context.Context, be restic.Backend, h restic.Handle) (bool, erro
|
|||
|
||||
// TestCreateWithConfig tests that creating a backend in a location which already
|
||||
// has a config file fails.
|
||||
func (s *Suite) TestCreateWithConfig(t *testing.T) {
|
||||
func (s *Suite[C]) TestCreateWithConfig(t *testing.T) {
|
||||
b := s.open(t)
|
||||
defer s.close(t, b)
|
||||
|
||||
|
@ -57,7 +57,7 @@ func (s *Suite) TestCreateWithConfig(t *testing.T) {
|
|||
store(t, b, restic.ConfigFile, []byte("test config"))
|
||||
|
||||
// now create the backend again, this must fail
|
||||
_, err = s.Create(s.Config)
|
||||
_, err = s.Create(*s.Config)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error not found for creating a backend with an existing config file")
|
||||
}
|
||||
|
@ -70,7 +70,7 @@ func (s *Suite) TestCreateWithConfig(t *testing.T) {
|
|||
}
|
||||
|
||||
// TestLocation tests that a location string is returned.
|
||||
func (s *Suite) TestLocation(t *testing.T) {
|
||||
func (s *Suite[C]) TestLocation(t *testing.T) {
|
||||
b := s.open(t)
|
||||
defer s.close(t, b)
|
||||
|
||||
|
@ -81,7 +81,7 @@ func (s *Suite) TestLocation(t *testing.T) {
|
|||
}
|
||||
|
||||
// TestConfig saves and loads a config from the backend.
|
||||
func (s *Suite) TestConfig(t *testing.T) {
|
||||
func (s *Suite[C]) TestConfig(t *testing.T) {
|
||||
b := s.open(t)
|
||||
defer s.close(t, b)
|
||||
|
||||
|
@ -118,7 +118,7 @@ func (s *Suite) TestConfig(t *testing.T) {
|
|||
}
|
||||
|
||||
// TestLoad tests the backend's Load function.
|
||||
func (s *Suite) TestLoad(t *testing.T) {
|
||||
func (s *Suite[C]) TestLoad(t *testing.T) {
|
||||
seedRand(t)
|
||||
|
||||
b := s.open(t)
|
||||
|
@ -222,8 +222,12 @@ func (s *Suite) TestLoad(t *testing.T) {
|
|||
test.OK(t, b.Remove(context.TODO(), handle))
|
||||
}
|
||||
|
||||
type setter interface {
|
||||
SetListMaxItems(int)
|
||||
}
|
||||
|
||||
// TestList makes sure that the backend implements List() pagination correctly.
|
||||
func (s *Suite) TestList(t *testing.T) {
|
||||
func (s *Suite[C]) TestList(t *testing.T) {
|
||||
seedRand(t)
|
||||
|
||||
numTestFiles := rand.Intn(20) + 20
|
||||
|
@ -269,10 +273,6 @@ func (s *Suite) TestList(t *testing.T) {
|
|||
t.Run(fmt.Sprintf("max-%v", test.maxItems), func(t *testing.T) {
|
||||
list2 := make(map[restic.ID]int64)
|
||||
|
||||
type setter interface {
|
||||
SetListMaxItems(int)
|
||||
}
|
||||
|
||||
if s, ok := b.(setter); ok {
|
||||
t.Logf("setting max list items to %d", test.maxItems)
|
||||
s.SetListMaxItems(test.maxItems)
|
||||
|
@ -326,7 +326,7 @@ func (s *Suite) TestList(t *testing.T) {
|
|||
}
|
||||
|
||||
// TestListCancel tests that the context is respected and the error is returned by List.
|
||||
func (s *Suite) TestListCancel(t *testing.T) {
|
||||
func (s *Suite[C]) TestListCancel(t *testing.T) {
|
||||
seedRand(t)
|
||||
|
||||
numTestFiles := 5
|
||||
|
@ -466,7 +466,7 @@ func (ec errorCloser) Rewind() error {
|
|||
}
|
||||
|
||||
// TestSave tests saving data in the backend.
|
||||
func (s *Suite) TestSave(t *testing.T) {
|
||||
func (s *Suite[C]) TestSave(t *testing.T) {
|
||||
seedRand(t)
|
||||
|
||||
b := s.open(t)
|
||||
|
@ -582,7 +582,7 @@ func (r *incompleteByteReader) Length() int64 {
|
|||
}
|
||||
|
||||
// TestSaveError tests saving data in the backend.
|
||||
func (s *Suite) TestSaveError(t *testing.T) {
|
||||
func (s *Suite[C]) TestSaveError(t *testing.T) {
|
||||
seedRand(t)
|
||||
|
||||
b := s.open(t)
|
||||
|
@ -621,7 +621,7 @@ func (b *wrongByteReader) Hash() []byte {
|
|||
}
|
||||
|
||||
// TestSaveWrongHash tests that uploads with a wrong hash fail
|
||||
func (s *Suite) TestSaveWrongHash(t *testing.T) {
|
||||
func (s *Suite[C]) TestSaveWrongHash(t *testing.T) {
|
||||
seedRand(t)
|
||||
|
||||
b := s.open(t)
|
||||
|
@ -679,7 +679,7 @@ func testLoad(b restic.Backend, h restic.Handle) error {
|
|||
})
|
||||
}
|
||||
|
||||
func (s *Suite) delayedRemove(t testing.TB, be restic.Backend, handles ...restic.Handle) error {
|
||||
func (s *Suite[C]) delayedRemove(t testing.TB, be restic.Backend, handles ...restic.Handle) error {
|
||||
// Some backend (swift, I'm looking at you) may implement delayed
|
||||
// removal of data. Let's wait a bit if this happens.
|
||||
|
||||
|
@ -746,7 +746,7 @@ func delayedList(t testing.TB, b restic.Backend, tpe restic.FileType, max int, m
|
|||
}
|
||||
|
||||
// TestBackend tests all functions of the backend.
|
||||
func (s *Suite) TestBackend(t *testing.T) {
|
||||
func (s *Suite[C]) TestBackend(t *testing.T) {
|
||||
b := s.open(t)
|
||||
defer s.close(t, b)
|
||||
|
||||
|
@ -867,7 +867,7 @@ func (s *Suite) TestBackend(t *testing.T) {
|
|||
}
|
||||
|
||||
// TestZZZDelete tests the Delete function. The name ensures that this test is executed last.
|
||||
func (s *Suite) TestZZZDelete(t *testing.T) {
|
||||
func (s *Suite[C]) TestZZZDelete(t *testing.T) {
|
||||
if !test.TestCleanupTempDirs {
|
||||
t.Skipf("not removing backend, TestCleanupTempDirs is false")
|
||||
}
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
package restorer
|
||||
package fs
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
@ -6,7 +6,7 @@ import (
|
|||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func preallocateFile(wr *os.File, size int64) error {
|
||||
func PreallocateFile(wr *os.File, size int64) error {
|
||||
// try contiguous first
|
||||
fst := unix.Fstore_t{
|
||||
Flags: unix.F_ALLOCATECONTIG | unix.F_ALLOCATEALL,
|
|
@ -1,4 +1,4 @@
|
|||
package restorer
|
||||
package fs
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
@ -6,7 +6,7 @@ import (
|
|||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func preallocateFile(wr *os.File, size int64) error {
|
||||
func PreallocateFile(wr *os.File, size int64) error {
|
||||
if size <= 0 {
|
||||
return nil
|
||||
}
|
|
@ -1,11 +1,11 @@
|
|||
//go:build !linux && !darwin
|
||||
// +build !linux,!darwin
|
||||
|
||||
package restorer
|
||||
package fs
|
||||
|
||||
import "os"
|
||||
|
||||
func preallocateFile(wr *os.File, size int64) error {
|
||||
func PreallocateFile(wr *os.File, size int64) error {
|
||||
// Maybe truncate can help?
|
||||
// Windows: This calls SetEndOfFile which preallocates space on disk
|
||||
return wr.Truncate(size)
|
|
@ -1,4 +1,4 @@
|
|||
package restorer
|
||||
package fs
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
@ -7,7 +7,6 @@ import (
|
|||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/fs"
|
||||
"github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
|
@ -23,7 +22,7 @@ func TestPreallocate(t *testing.T) {
|
|||
test.OK(t, wr.Close())
|
||||
}()
|
||||
|
||||
err = preallocateFile(wr, i)
|
||||
err = PreallocateFile(wr, i)
|
||||
if err == syscall.ENOTSUP {
|
||||
t.SkipNow()
|
||||
}
|
||||
|
@ -32,7 +31,7 @@ func TestPreallocate(t *testing.T) {
|
|||
fi, err := wr.Stat()
|
||||
test.OK(t, err)
|
||||
|
||||
efi := fs.ExtendedStat(fi)
|
||||
efi := ExtendedStat(fi)
|
||||
test.Assert(t, efi.Size == i || efi.Blocks > 0, "Preallocated size of %v, got size %v block %v", i, efi.Size, efi.Blocks)
|
||||
})
|
||||
}
|
|
@ -80,3 +80,8 @@ type FileInfo struct {
|
|||
Size int64
|
||||
Name string
|
||||
}
|
||||
|
||||
// ApplyEnvironmenter fills in a backend configuration from the environment
|
||||
type ApplyEnvironmenter interface {
|
||||
ApplyEnvironment(prefix string) error
|
||||
}
|
||||
|
|
|
@ -6,6 +6,7 @@ import (
|
|||
|
||||
"github.com/cespare/xxhash/v2"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/fs"
|
||||
)
|
||||
|
||||
// writes blobs to target files.
|
||||
|
@ -72,7 +73,7 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create
|
|||
return nil, err
|
||||
}
|
||||
} else {
|
||||
err := preallocateFile(wr.File, createSize)
|
||||
err := fs.PreallocateFile(wr.File, createSize)
|
||||
if err != nil {
|
||||
// Just log the preallocate error but don't let it cause the restore process to fail.
|
||||
// Preallocate might return an error if the filesystem (implementation) does not
|
||||
|
|
|
@ -44,6 +44,7 @@ type Progress struct {
|
|||
mu sync.Mutex
|
||||
|
||||
start time.Time
|
||||
estimator rateEstimator
|
||||
|
||||
scanStarted, scanFinished bool
|
||||
|
||||
|
@ -60,6 +61,7 @@ func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress {
|
|||
start: time.Now(),
|
||||
currentFiles: make(map[string]struct{}),
|
||||
printer: printer,
|
||||
estimator: *newRateEstimator(time.Now()),
|
||||
}
|
||||
p.Updater = *progress.NewUpdater(interval, func(runtime time.Duration, final bool) {
|
||||
if final {
|
||||
|
@ -73,9 +75,14 @@ func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress {
|
|||
|
||||
var secondsRemaining uint64
|
||||
if p.scanFinished {
|
||||
secs := float64(runtime / time.Second)
|
||||
rate := p.estimator.rate(time.Now())
|
||||
tooSlowCutoff := 1024.
|
||||
if rate <= tooSlowCutoff {
|
||||
secondsRemaining = 0
|
||||
} else {
|
||||
todo := float64(p.total.Bytes - p.processed.Bytes)
|
||||
secondsRemaining = uint64(secs / float64(p.processed.Bytes) * todo)
|
||||
secondsRemaining = uint64(todo / rate)
|
||||
}
|
||||
}
|
||||
|
||||
p.printer.Update(p.total, p.processed, p.errors, p.currentFiles, p.start, secondsRemaining)
|
||||
|
@ -105,6 +112,7 @@ func (p *Progress) addProcessed(c Counter) {
|
|||
p.processed.Files += c.Files
|
||||
p.processed.Dirs += c.Dirs
|
||||
p.processed.Bytes += c.Bytes
|
||||
p.estimator.recordBytes(time.Now(), c.Bytes)
|
||||
p.scanStarted = true
|
||||
}
|
||||
|
||||
|
|
98
internal/ui/backup/rate_estimator.go
Normal file
98
internal/ui/backup/rate_estimator.go
Normal file
|
@ -0,0 +1,98 @@
|
|||
package backup
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"time"
|
||||
)
|
||||
|
||||
// rateBucket represents a one second window of recorded progress.
|
||||
type rateBucket struct {
|
||||
totalBytes uint64
|
||||
end time.Time // the end of the time window, exclusive
|
||||
}
|
||||
|
||||
// rateEstimator represents an estimate of the time to complete an operation.
|
||||
type rateEstimator struct {
|
||||
buckets *list.List
|
||||
start time.Time
|
||||
totalBytes uint64
|
||||
}
|
||||
|
||||
// newRateEstimator returns an estimator initialized to a presumed start time.
|
||||
func newRateEstimator(start time.Time) *rateEstimator {
|
||||
return &rateEstimator{buckets: list.New(), start: start}
|
||||
}
|
||||
|
||||
// See trim(), below.
|
||||
const (
|
||||
bucketWidth = time.Second
|
||||
minRateEstimatorBytes = 100 * 1000 * 1000
|
||||
minRateEstimatorBuckets = 20
|
||||
minRateEstimatorMinutes = 2
|
||||
)
|
||||
|
||||
// trim removes the oldest history from the estimator assuming a given
|
||||
// current time.
|
||||
func (r *rateEstimator) trim(now time.Time) {
|
||||
// The estimator retains byte transfer counts over a two minute window.
|
||||
// However, to avoid removing too much history when transfer rates are
|
||||
// low, the estimator also retains a minimum number of processed bytes
|
||||
// across a minimum number of buckets. An operation that is processing a
|
||||
// significant number of bytes per second will typically retain only a
|
||||
// two minute window's worth of information. One that is making slow
|
||||
// progress, such as one being over a rate limited connection, typically
|
||||
// observes bursts of updates as infrequently as every ten or twenty
|
||||
// seconds, in which case the other limiters will kick in. This heuristic
|
||||
// avoids wildly fluctuating estimates over rate limited connections.
|
||||
start := now.Add(-minRateEstimatorMinutes * time.Minute)
|
||||
|
||||
for e := r.buckets.Front(); e != nil; e = r.buckets.Front() {
|
||||
if r.buckets.Len() <= minRateEstimatorBuckets {
|
||||
break
|
||||
}
|
||||
b := e.Value.(*rateBucket)
|
||||
if b.end.After(start) {
|
||||
break
|
||||
}
|
||||
total := r.totalBytes - b.totalBytes
|
||||
if total < minRateEstimatorBytes {
|
||||
break
|
||||
}
|
||||
r.start = b.end
|
||||
r.totalBytes = total
|
||||
r.buckets.Remove(e)
|
||||
}
|
||||
}
|
||||
|
||||
// recordBytes records the transfer of a number of bytes at a given
|
||||
// time. Times passed in successive calls should advance monotonically (as
|
||||
// is the case with time.Now().
|
||||
func (r *rateEstimator) recordBytes(now time.Time, bytes uint64) {
|
||||
if bytes == 0 {
|
||||
return
|
||||
}
|
||||
var tail *rateBucket
|
||||
if r.buckets.Len() > 0 {
|
||||
tail = r.buckets.Back().Value.(*rateBucket)
|
||||
}
|
||||
if tail == nil || !tail.end.After(now) {
|
||||
// The new bucket holds measurements in the time range [now .. now+1sec).
|
||||
tail = &rateBucket{end: now.Add(bucketWidth)}
|
||||
r.buckets.PushBack(tail)
|
||||
}
|
||||
tail.totalBytes += bytes
|
||||
r.totalBytes += bytes
|
||||
r.trim(now)
|
||||
}
|
||||
|
||||
// rate returns an estimated bytes per second rate at a given time, or zero
|
||||
// if there is not enough data to compute a rate.
|
||||
func (r *rateEstimator) rate(now time.Time) float64 {
|
||||
r.trim(now)
|
||||
if !r.start.Before(now) {
|
||||
return 0
|
||||
}
|
||||
elapsed := float64(now.Sub(r.start)) / float64(time.Second)
|
||||
rate := float64(r.totalBytes) / elapsed
|
||||
return rate
|
||||
}
|
213
internal/ui/backup/rate_estimator_test.go
Normal file
213
internal/ui/backup/rate_estimator_test.go
Normal file
|
@ -0,0 +1,213 @@
|
|||
package backup
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
const float64EqualityThreshold = 1e-6
|
||||
|
||||
func almostEqual(a, b float64) bool {
|
||||
if math.IsNaN(a) || math.IsNaN(b) {
|
||||
panic("almostEqual passed a NaN")
|
||||
}
|
||||
return math.Abs(a-b) <= float64EqualityThreshold
|
||||
}
|
||||
|
||||
func TestEstimatorDefault(t *testing.T) {
|
||||
var start time.Time
|
||||
e := newRateEstimator(start)
|
||||
r := e.rate(start)
|
||||
rtest.Assert(t, r == 0, "e.Rate == %v, want zero", r)
|
||||
r = e.rate(start.Add(time.Hour))
|
||||
rtest.Assert(t, r == 0, "e.Rate == %v, want zero", r)
|
||||
}
|
||||
|
||||
func TestEstimatorSimple(t *testing.T) {
|
||||
var start time.Time
|
||||
type testcase struct {
|
||||
bytes uint64
|
||||
when time.Duration
|
||||
rate float64
|
||||
}
|
||||
|
||||
cases := []testcase{
|
||||
{0, 0, 0},
|
||||
{1, time.Second, 1},
|
||||
{60, time.Second, 60},
|
||||
{60, time.Minute, 1},
|
||||
}
|
||||
for _, c := range cases {
|
||||
name := fmt.Sprintf("%+v", c)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
e := newRateEstimator(start)
|
||||
e.recordBytes(start.Add(time.Second), c.bytes)
|
||||
rate := e.rate(start.Add(c.when))
|
||||
rtest.Assert(t, almostEqual(rate, c.rate), "e.Rate == %v, want %v", rate, c.rate)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketWidth(t *testing.T) {
|
||||
var when time.Time
|
||||
|
||||
// Recording byte transfers within a bucket width's time window uses one
|
||||
// bucket.
|
||||
e := newRateEstimator(when)
|
||||
e.recordBytes(when, 1)
|
||||
e.recordBytes(when.Add(bucketWidth-time.Nanosecond), 1)
|
||||
rtest.Assert(t, e.buckets.Len() == 1, "e.buckets.Len() is %d, want 1", e.buckets.Len())
|
||||
|
||||
b := e.buckets.Back().Value.(*rateBucket)
|
||||
rtest.Assert(t, b.totalBytes == 2, "b.totalBytes is %d, want 2", b.totalBytes)
|
||||
rtest.Assert(t, b.end == when.Add(bucketWidth), "b.end is %v, want %v", b.end, when.Add(bucketWidth))
|
||||
|
||||
// Recording a byte outside the bucket width causes another bucket.
|
||||
e.recordBytes(when.Add(bucketWidth), 1)
|
||||
rtest.Assert(t, e.buckets.Len() == 2, "e.buckets.Len() is %d, want 2", e.buckets.Len())
|
||||
|
||||
b = e.buckets.Back().Value.(*rateBucket)
|
||||
rtest.Assert(t, b.totalBytes == 1, "b.totalBytes is %d, want 1", b.totalBytes)
|
||||
rtest.Assert(t, b.end == when.Add(2*bucketWidth), "b.end is %v, want %v", b.end, when.Add(bucketWidth))
|
||||
|
||||
// Recording a byte after a longer delay creates a sparse bucket list.
|
||||
e.recordBytes(when.Add(time.Hour+time.Millisecond), 7)
|
||||
rtest.Assert(t, e.buckets.Len() == 3, "e.buckets.Len() is %d, want 3", e.buckets.Len())
|
||||
|
||||
b = e.buckets.Back().Value.(*rateBucket)
|
||||
rtest.Assert(t, b.totalBytes == 7, "b.totalBytes is %d, want 7", b.totalBytes)
|
||||
rtest.Equals(t, when.Add(time.Hour+time.Millisecond+time.Second), b.end)
|
||||
}
|
||||
|
||||
type chunk struct {
|
||||
repetitions uint64 // repetition count
|
||||
bytes uint64 // byte count (every second)
|
||||
}
|
||||
|
||||
func applyChunks(chunks []chunk, t time.Time, e *rateEstimator) time.Time {
|
||||
for _, c := range chunks {
|
||||
for i := uint64(0); i < c.repetitions; i++ {
|
||||
e.recordBytes(t, c.bytes)
|
||||
t = t.Add(time.Second)
|
||||
}
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func TestEstimatorResponsiveness(t *testing.T) {
|
||||
type testcase struct {
|
||||
description string
|
||||
chunks []chunk
|
||||
rate float64
|
||||
}
|
||||
|
||||
cases := []testcase{
|
||||
{
|
||||
"1000 bytes/sec over one second",
|
||||
[]chunk{
|
||||
{1, 1000},
|
||||
},
|
||||
1000,
|
||||
},
|
||||
{
|
||||
"1000 bytes/sec over one minute",
|
||||
[]chunk{
|
||||
{60, 1000},
|
||||
},
|
||||
1000,
|
||||
},
|
||||
{
|
||||
"1000 bytes/sec for 10 seconds, then 2000 bytes/sec for 10 seconds",
|
||||
[]chunk{
|
||||
{10, 1000},
|
||||
{10, 2000},
|
||||
},
|
||||
1500,
|
||||
},
|
||||
{
|
||||
"1000 bytes/sec for one minute, then 2000 bytes/sec for one minute",
|
||||
[]chunk{
|
||||
{60, 1000},
|
||||
{60, 2000},
|
||||
},
|
||||
1500,
|
||||
},
|
||||
{
|
||||
"rate doubles after 30 seconds",
|
||||
[]chunk{
|
||||
{30, minRateEstimatorBytes},
|
||||
{90, 2 * minRateEstimatorBytes},
|
||||
},
|
||||
minRateEstimatorBytes * 1.75,
|
||||
},
|
||||
{
|
||||
"rate doubles after 31 seconds",
|
||||
[]chunk{
|
||||
{31, minRateEstimatorBytes},
|
||||
{90, 2 * minRateEstimatorBytes},
|
||||
},
|
||||
// The expected rate is the same as the prior test case because the
|
||||
// first second has rolled off the estimator.
|
||||
minRateEstimatorBytes * 1.75,
|
||||
},
|
||||
{
|
||||
"rate doubles after 90 seconds",
|
||||
[]chunk{
|
||||
{90, minRateEstimatorBytes},
|
||||
{90, 2 * minRateEstimatorBytes},
|
||||
},
|
||||
// The expected rate is the same as the prior test case because the
|
||||
// first 60 seconds have rolled off the estimator.
|
||||
minRateEstimatorBytes * 1.75,
|
||||
},
|
||||
{
|
||||
"rate doubles for two full minutes",
|
||||
[]chunk{
|
||||
{60, minRateEstimatorBytes},
|
||||
{120, 2 * minRateEstimatorBytes},
|
||||
},
|
||||
2 * minRateEstimatorBytes,
|
||||
},
|
||||
{
|
||||
"rate falls to zero",
|
||||
[]chunk{
|
||||
{30, minRateEstimatorBytes},
|
||||
{30, 0},
|
||||
},
|
||||
minRateEstimatorBytes / 2,
|
||||
},
|
||||
{
|
||||
"rate falls to zero for extended time",
|
||||
[]chunk{
|
||||
{60, 1000},
|
||||
{300, 0},
|
||||
},
|
||||
1000 * 60 / (60 + 300.0),
|
||||
},
|
||||
{
|
||||
"rate falls to zero for extended time (from high rate)",
|
||||
[]chunk{
|
||||
{2 * minRateEstimatorBuckets, minRateEstimatorBytes},
|
||||
{300, 0},
|
||||
},
|
||||
// Expect that only minRateEstimatorBuckets buckets are used in the
|
||||
// rate estimate.
|
||||
minRateEstimatorBytes * minRateEstimatorBuckets /
|
||||
(minRateEstimatorBuckets + 300.0),
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.description, func(t *testing.T) {
|
||||
var w time.Time
|
||||
e := newRateEstimator(w)
|
||||
w = applyChunks(c.chunks, w, e)
|
||||
r := e.rate(w)
|
||||
rtest.Assert(t, almostEqual(r, c.rate), "e.Rate == %f, want %f", r, c.rate)
|
||||
})
|
||||
}
|
||||
}
|
Loading…
Add table
Reference in a new issue