1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2025-03-16 00:00:05 +01:00

Merge remote-tracking branch 'old-origin/master' into add-smb-backend

This commit is contained in:
Aneesh Nireshwalia 2023-07-10 11:17:50 -06:00
commit 4275870876
50 changed files with 966 additions and 758 deletions

View file

@ -0,0 +1,11 @@
Enhancement: Improve the ETA displayed during backup
Restic's `backup` command displayed an ETA that did not adapt when the rate
of progress made during the backup changed during the course of the
backup. Restic now uses recent progress when computing the ETA. It is
important to realize that the estimate may still be wrong, because restic
cannot predict the future, but the hope is that the ETA will be more
accurate in most cases.
https://github.com/restic/restic/issues/3397
https://github.com/restic/restic/pull/3563

View file

@ -0,0 +1,8 @@
Enhancement: Reduce file fragmentation for local backend
Before this change, local backend files could become fragmented.
Now restic will try to preallocate space for pack files to avoid
their fragmentation.
https://github.com/restic/restic/issues/2679
https://github.com/restic/restic/pull/3261

View file

@ -536,170 +536,21 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi
} }
func parseConfig(loc location.Location, opts options.Options) (interface{}, error) { func parseConfig(loc location.Location, opts options.Options) (interface{}, error) {
// only apply options for a particular backend here cfg := loc.Config
opts = opts.Extract(loc.Scheme) if cfg, ok := cfg.(restic.ApplyEnvironmenter); ok {
if err := cfg.ApplyEnvironment(""); err != nil {
switch loc.Scheme {
case "local":
cfg := loc.Config.(local.Config)
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
return nil, err return nil, err
} }
debug.Log("opening local repository at %#v", cfg)
return cfg, nil
case "sftp":
cfg := loc.Config.(sftp.Config)
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
return nil, err
}
debug.Log("opening sftp repository at %#v", cfg)
return cfg, nil
case "s3":
cfg := loc.Config.(s3.Config)
if cfg.KeyID == "" {
cfg.KeyID = os.Getenv("AWS_ACCESS_KEY_ID")
}
if cfg.Secret.String() == "" {
cfg.Secret = options.NewSecretString(os.Getenv("AWS_SECRET_ACCESS_KEY"))
}
if cfg.KeyID == "" && cfg.Secret.String() != "" {
return nil, errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty")
} else if cfg.KeyID != "" && cfg.Secret.String() == "" {
return nil, errors.Fatalf("unable to open S3 backend: Secret ($AWS_SECRET_ACCESS_KEY) is empty")
}
if cfg.Region == "" {
cfg.Region = os.Getenv("AWS_DEFAULT_REGION")
}
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
return nil, err
}
debug.Log("opening s3 repository at %#v", cfg)
return cfg, nil
case "gs":
cfg := loc.Config.(gs.Config)
if cfg.ProjectID == "" {
cfg.ProjectID = os.Getenv("GOOGLE_PROJECT_ID")
}
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
return nil, err
}
debug.Log("opening gs repository at %#v", cfg)
return cfg, nil
case "azure":
cfg := loc.Config.(azure.Config)
if cfg.AccountName == "" {
cfg.AccountName = os.Getenv("AZURE_ACCOUNT_NAME")
}
if cfg.AccountKey.String() == "" {
cfg.AccountKey = options.NewSecretString(os.Getenv("AZURE_ACCOUNT_KEY"))
}
if cfg.AccountSAS.String() == "" {
cfg.AccountSAS = options.NewSecretString(os.Getenv("AZURE_ACCOUNT_SAS"))
}
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
return nil, err
}
debug.Log("opening gs repository at %#v", cfg)
return cfg, nil
case "swift":
cfg := loc.Config.(swift.Config)
if err := swift.ApplyEnvironment("", &cfg); err != nil {
return nil, err
}
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
return nil, err
}
debug.Log("opening swift repository at %#v", cfg)
return cfg, nil
case "b2":
cfg := loc.Config.(b2.Config)
if cfg.AccountID == "" {
cfg.AccountID = os.Getenv("B2_ACCOUNT_ID")
}
if cfg.AccountID == "" {
return nil, errors.Fatalf("unable to open B2 backend: Account ID ($B2_ACCOUNT_ID) is empty")
}
if cfg.Key.String() == "" {
cfg.Key = options.NewSecretString(os.Getenv("B2_ACCOUNT_KEY"))
}
if cfg.Key.String() == "" {
return nil, errors.Fatalf("unable to open B2 backend: Key ($B2_ACCOUNT_KEY) is empty")
}
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
return nil, err
}
debug.Log("opening b2 repository at %#v", cfg)
return cfg, nil
case "rest":
cfg := loc.Config.(rest.Config)
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
return nil, err
}
debug.Log("opening rest repository at %#v", cfg)
return cfg, nil
case "rclone":
cfg := loc.Config.(rclone.Config)
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
return nil, err
}
debug.Log("opening rest repository at %#v", cfg)
return cfg, nil
case "smb":
cfg := loc.Config.(smb.Config)
if cfg.User == "" {
cfg.User = os.Getenv("RESTIC_SMB_USER")
}
if cfg.Password.String() == "" {
cfg.Password = options.NewSecretString(os.Getenv("RESTIC_SMB_PASSWORD"))
}
if cfg.Domain == "" {
cfg.Domain = os.Getenv("RESTIC_SMB_DOMAIN")
}
if cfg.Domain == "" {
cfg.Domain = smb.DefaultDomain
}
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
return nil, err
}
debug.Log("opening smb repository at %#v", cfg)
return cfg, nil
} }
return nil, errors.Fatalf("invalid backend: %q", loc.Scheme) // only apply options for a particular backend here
opts = opts.Extract(loc.Scheme)
if err := opts.Apply(loc.Scheme, cfg); err != nil {
return nil, err
}
debug.Log("opening %v repository at %#v", loc.Scheme, cfg)
return cfg, nil
} }
// Open the backend specified by a location config. // Open the backend specified by a location config.
@ -728,25 +579,25 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio
switch loc.Scheme { switch loc.Scheme {
case "local": case "local":
be, err = local.Open(ctx, cfg.(local.Config)) be, err = local.Open(ctx, *cfg.(*local.Config))
case "sftp": case "sftp":
be, err = sftp.Open(ctx, cfg.(sftp.Config)) be, err = sftp.Open(ctx, *cfg.(*sftp.Config))
case "smb":
be, err = smb.Open(ctx, cfg.(smb.Config))
case "s3": case "s3":
be, err = s3.Open(ctx, cfg.(s3.Config), rt) be, err = s3.Open(ctx, *cfg.(*s3.Config), rt)
case "gs": case "gs":
be, err = gs.Open(cfg.(gs.Config), rt) be, err = gs.Open(*cfg.(*gs.Config), rt)
case "azure": case "azure":
be, err = azure.Open(ctx, cfg.(azure.Config), rt) be, err = azure.Open(ctx, *cfg.(*azure.Config), rt)
case "swift": case "swift":
be, err = swift.Open(ctx, cfg.(swift.Config), rt) be, err = swift.Open(ctx, *cfg.(*swift.Config), rt)
case "b2": case "b2":
be, err = b2.Open(ctx, cfg.(b2.Config), rt) be, err = b2.Open(ctx, *cfg.(*b2.Config), rt)
case "rest": case "rest":
be, err = rest.Open(cfg.(rest.Config), rt) be, err = rest.Open(*cfg.(*rest.Config), rt)
case "rclone": case "rclone":
be, err = rclone.Open(cfg.(rclone.Config), lim) be, err = rclone.Open(*cfg.(*rclone.Config), lim)
case "smb":
be, err = smb.Open(ctx, *cfg.(*smb.Config))
default: default:
return nil, errors.Fatalf("invalid backend: %q", loc.Scheme) return nil, errors.Fatalf("invalid backend: %q", loc.Scheme)
@ -806,25 +657,25 @@ func create(ctx context.Context, s string, opts options.Options) (restic.Backend
var be restic.Backend var be restic.Backend
switch loc.Scheme { switch loc.Scheme {
case "local": case "local":
be, err = local.Create(ctx, cfg.(local.Config)) be, err = local.Create(ctx, *cfg.(*local.Config))
case "sftp": case "sftp":
be, err = sftp.Create(ctx, cfg.(sftp.Config)) be, err = sftp.Create(ctx, *cfg.(*sftp.Config))
case "smb":
be, err = smb.Create(ctx, cfg.(smb.Config))
case "s3": case "s3":
be, err = s3.Create(ctx, cfg.(s3.Config), rt) be, err = s3.Create(ctx, *cfg.(*s3.Config), rt)
case "gs": case "gs":
be, err = gs.Create(ctx, cfg.(gs.Config), rt) be, err = gs.Create(ctx, *cfg.(*gs.Config), rt)
case "azure": case "azure":
be, err = azure.Create(ctx, cfg.(azure.Config), rt) be, err = azure.Create(ctx, *cfg.(*azure.Config), rt)
case "swift": case "swift":
be, err = swift.Open(ctx, cfg.(swift.Config), rt) be, err = swift.Open(ctx, *cfg.(*swift.Config), rt)
case "b2": case "b2":
be, err = b2.Create(ctx, cfg.(b2.Config), rt) be, err = b2.Create(ctx, *cfg.(*b2.Config), rt)
case "rest": case "rest":
be, err = rest.Create(ctx, cfg.(rest.Config), rt) be, err = rest.Create(ctx, *cfg.(*rest.Config), rt)
case "rclone": case "rclone":
be, err = rclone.Create(ctx, cfg.(rclone.Config)) be, err = rclone.Create(ctx, *cfg.(*rclone.Config))
case "smb":
be, err = smb.Create(ctx, *cfg.(*smb.Config))
default: default:
debug.Log("invalid repository scheme: %v", s) debug.Log("invalid repository scheme: %v", s)
return nil, errors.Fatalf("invalid scheme %q", loc.Scheme) return nil, errors.Fatalf("invalid scheme %q", loc.Scheme)

View file

@ -18,34 +18,34 @@ import (
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
func newAzureTestSuite(t testing.TB) *test.Suite { func newAzureTestSuite(t testing.TB) *test.Suite[azure.Config] {
tr, err := backend.Transport(backend.TransportOptions{}) tr, err := backend.Transport(backend.TransportOptions{})
if err != nil { if err != nil {
t.Fatalf("cannot create transport for tests: %v", err) t.Fatalf("cannot create transport for tests: %v", err)
} }
return &test.Suite{ return &test.Suite[azure.Config]{
// do not use excessive data // do not use excessive data
MinimalData: true, MinimalData: true,
// NewConfig returns a config for a new temporary backend that will be used in tests. // NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) { NewConfig: func() (*azure.Config, error) {
azcfg, err := azure.ParseConfig(os.Getenv("RESTIC_TEST_AZURE_REPOSITORY")) cfg, err := azure.ParseConfig(os.Getenv("RESTIC_TEST_AZURE_REPOSITORY"))
if err != nil {
return nil, err
}
err = cfg.ApplyEnvironment("RESTIC_TEST_")
if err != nil { if err != nil {
return nil, err return nil, err
} }
cfg := azcfg.(azure.Config)
cfg.AccountName = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_NAME")
cfg.AccountKey = options.NewSecretString(os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY"))
cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
return cfg, nil return cfg, nil
}, },
// CreateFn is a function that creates a temporary repository for the tests. // CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) { Create: func(cfg azure.Config) (restic.Backend, error) {
cfg := config.(azure.Config)
ctx := context.TODO() ctx := context.TODO()
be, err := azure.Create(ctx, cfg, tr) be, err := azure.Create(ctx, cfg, tr)
if err != nil { if err != nil {
@ -65,15 +65,13 @@ func newAzureTestSuite(t testing.TB) *test.Suite {
}, },
// OpenFn is a function that opens a previously created temporary repository. // OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) { Open: func(cfg azure.Config) (restic.Backend, error) {
cfg := config.(azure.Config)
ctx := context.TODO() ctx := context.TODO()
return azure.Open(ctx, cfg, tr) return azure.Open(ctx, cfg, tr)
}, },
// CleanupFn removes data created during the tests. // CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error { Cleanup: func(cfg azure.Config) error {
cfg := config.(azure.Config)
ctx := context.TODO() ctx := context.TODO()
be, err := azure.Open(ctx, cfg, tr) be, err := azure.Open(ctx, cfg, tr)
if err != nil { if err != nil {
@ -141,12 +139,11 @@ func TestUploadLargeFile(t *testing.T) {
return return
} }
azcfg, err := azure.ParseConfig(os.Getenv("RESTIC_TEST_AZURE_REPOSITORY")) cfg, err := azure.ParseConfig(os.Getenv("RESTIC_TEST_AZURE_REPOSITORY"))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
cfg := azcfg.(azure.Config)
cfg.AccountName = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_NAME") cfg.AccountName = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_NAME")
cfg.AccountKey = options.NewSecretString(os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY")) cfg.AccountKey = options.NewSecretString(os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY"))
cfg.Prefix = fmt.Sprintf("test-upload-large-%d", time.Now().UnixNano()) cfg.Prefix = fmt.Sprintf("test-upload-large-%d", time.Now().UnixNano())
@ -156,7 +153,7 @@ func TestUploadLargeFile(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
be, err := azure.Create(ctx, cfg, tr) be, err := azure.Create(ctx, *cfg, tr)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View file

@ -1,11 +1,13 @@
package azure package azure
import ( import (
"os"
"path" "path"
"strings" "strings"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options" "github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
) )
// Config contains all configuration necessary to connect to an azure compatible // Config contains all configuration necessary to connect to an azure compatible
@ -33,7 +35,7 @@ func init() {
// ParseConfig parses the string s and extracts the azure config. The // ParseConfig parses the string s and extracts the azure config. The
// configuration format is azure:containerName:/[prefix]. // configuration format is azure:containerName:/[prefix].
func ParseConfig(s string) (interface{}, error) { func ParseConfig(s string) (*Config, error) {
if !strings.HasPrefix(s, "azure:") { if !strings.HasPrefix(s, "azure:") {
return nil, errors.New("azure: invalid format") return nil, errors.New("azure: invalid format")
} }
@ -51,5 +53,23 @@ func ParseConfig(s string) (interface{}, error) {
cfg := NewConfig() cfg := NewConfig()
cfg.Container = container cfg.Container = container
cfg.Prefix = prefix cfg.Prefix = prefix
return cfg, nil return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) error {
if cfg.AccountName == "" {
cfg.AccountName = os.Getenv(prefix + "AZURE_ACCOUNT_NAME")
}
if cfg.AccountKey.String() == "" {
cfg.AccountKey = options.NewSecretString(os.Getenv(prefix + "AZURE_ACCOUNT_KEY"))
}
if cfg.AccountSAS.String() == "" {
cfg.AccountSAS = options.NewSecretString(os.Getenv(prefix + "AZURE_ACCOUNT_SAS"))
}
return nil
} }

View file

@ -1,22 +1,23 @@
package azure package azure
import "testing" import (
"testing"
var configTests = []struct { "github.com/restic/restic/internal/backend/test"
s string )
cfg Config
}{ var configTests = []test.ConfigTestData[Config]{
{"azure:container-name:/", Config{ {S: "azure:container-name:/", Cfg: Config{
Container: "container-name", Container: "container-name",
Prefix: "", Prefix: "",
Connections: 5, Connections: 5,
}}, }},
{"azure:container-name:/prefix/directory", Config{ {S: "azure:container-name:/prefix/directory", Cfg: Config{
Container: "container-name", Container: "container-name",
Prefix: "prefix/directory", Prefix: "prefix/directory",
Connections: 5, Connections: 5,
}}, }},
{"azure:container-name:/prefix/directory/", Config{ {S: "azure:container-name:/prefix/directory/", Cfg: Config{
Container: "container-name", Container: "container-name",
Prefix: "prefix/directory", Prefix: "prefix/directory",
Connections: 5, Connections: 5,
@ -24,17 +25,5 @@ var configTests = []struct {
} }
func TestParseConfig(t *testing.T) { func TestParseConfig(t *testing.T) {
for i, test := range configTests { test.ParseConfigTester(t, ParseConfig, configTests)
cfg, err := ParseConfig(test.s)
if err != nil {
t.Errorf("test %d:%s failed: %v", i, test.s, err)
continue
}
if cfg != test.cfg {
t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v",
i, test.s, test.cfg, cfg)
continue
}
}
} }

View file

@ -10,19 +10,18 @@ import (
"github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/b2" "github.com/restic/restic/internal/backend/b2"
"github.com/restic/restic/internal/backend/test" "github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
func newB2TestSuite(t testing.TB) *test.Suite { func newB2TestSuite(t testing.TB) *test.Suite[b2.Config] {
tr, err := backend.Transport(backend.TransportOptions{}) tr, err := backend.Transport(backend.TransportOptions{})
if err != nil { if err != nil {
t.Fatalf("cannot create transport for tests: %v", err) t.Fatalf("cannot create transport for tests: %v", err)
} }
return &test.Suite{ return &test.Suite[b2.Config]{
// do not use excessive data // do not use excessive data
MinimalData: true, MinimalData: true,
@ -30,34 +29,33 @@ func newB2TestSuite(t testing.TB) *test.Suite {
WaitForDelayedRemoval: 10 * time.Second, WaitForDelayedRemoval: 10 * time.Second,
// NewConfig returns a config for a new temporary backend that will be used in tests. // NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) { NewConfig: func() (*b2.Config, error) {
b2cfg, err := b2.ParseConfig(os.Getenv("RESTIC_TEST_B2_REPOSITORY")) cfg, err := b2.ParseConfig(os.Getenv("RESTIC_TEST_B2_REPOSITORY"))
if err != nil {
return nil, err
}
err = cfg.ApplyEnvironment("RESTIC_TEST_")
if err != nil { if err != nil {
return nil, err return nil, err
} }
cfg := b2cfg.(b2.Config)
cfg.AccountID = os.Getenv("RESTIC_TEST_B2_ACCOUNT_ID")
cfg.Key = options.NewSecretString(os.Getenv("RESTIC_TEST_B2_ACCOUNT_KEY"))
cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
return cfg, nil return cfg, nil
}, },
// CreateFn is a function that creates a temporary repository for the tests. // CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) { Create: func(cfg b2.Config) (restic.Backend, error) {
cfg := config.(b2.Config)
return b2.Create(context.Background(), cfg, tr) return b2.Create(context.Background(), cfg, tr)
}, },
// OpenFn is a function that opens a previously created temporary repository. // OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) { Open: func(cfg b2.Config) (restic.Backend, error) {
cfg := config.(b2.Config)
return b2.Open(context.Background(), cfg, tr) return b2.Open(context.Background(), cfg, tr)
}, },
// CleanupFn removes data created during the tests. // CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error { Cleanup: func(cfg b2.Config) error {
cfg := config.(b2.Config)
be, err := b2.Open(context.Background(), cfg, tr) be, err := b2.Open(context.Background(), cfg, tr)
if err != nil { if err != nil {
return err return err

View file

@ -1,12 +1,14 @@
package b2 package b2
import ( import (
"os"
"path" "path"
"regexp" "regexp"
"strings" "strings"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options" "github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
) )
// Config contains all configuration necessary to connect to an b2 compatible // Config contains all configuration necessary to connect to an b2 compatible
@ -58,7 +60,7 @@ func checkBucketName(name string) error {
// ParseConfig parses the string s and extracts the b2 config. The supported // ParseConfig parses the string s and extracts the b2 config. The supported
// configuration format is b2:bucketname/prefix. If no prefix is given the // configuration format is b2:bucketname/prefix. If no prefix is given the
// prefix "restic" will be used. // prefix "restic" will be used.
func ParseConfig(s string) (interface{}, error) { func ParseConfig(s string) (*Config, error) {
if !strings.HasPrefix(s, "b2:") { if !strings.HasPrefix(s, "b2:") {
return nil, errors.New("invalid format, want: b2:bucket-name[:path]") return nil, errors.New("invalid format, want: b2:bucket-name[:path]")
} }
@ -77,5 +79,27 @@ func ParseConfig(s string) (interface{}, error) {
cfg.Bucket = bucket cfg.Bucket = bucket
cfg.Prefix = prefix cfg.Prefix = prefix
return cfg, nil return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) error {
if cfg.AccountID == "" {
cfg.AccountID = os.Getenv(prefix + "B2_ACCOUNT_ID")
}
if cfg.AccountID == "" {
return errors.Fatalf("unable to open B2 backend: Account ID ($B2_ACCOUNT_ID) is empty")
}
if cfg.Key.String() == "" {
cfg.Key = options.NewSecretString(os.Getenv(prefix + "B2_ACCOUNT_KEY"))
}
if cfg.Key.String() == "" {
return errors.Fatalf("unable to open B2 backend: Key ($B2_ACCOUNT_KEY) is empty")
}
return nil
} }

View file

@ -1,37 +1,38 @@
package b2 package b2
import "testing" import (
"testing"
var configTests = []struct { "github.com/restic/restic/internal/backend/test"
s string )
cfg Config
}{ var configTests = []test.ConfigTestData[Config]{
{"b2:bucketname", Config{ {S: "b2:bucketname", Cfg: Config{
Bucket: "bucketname", Bucket: "bucketname",
Prefix: "", Prefix: "",
Connections: 5, Connections: 5,
}}, }},
{"b2:bucketname:", Config{ {S: "b2:bucketname:", Cfg: Config{
Bucket: "bucketname", Bucket: "bucketname",
Prefix: "", Prefix: "",
Connections: 5, Connections: 5,
}}, }},
{"b2:bucketname:/prefix/directory", Config{ {S: "b2:bucketname:/prefix/directory", Cfg: Config{
Bucket: "bucketname", Bucket: "bucketname",
Prefix: "prefix/directory", Prefix: "prefix/directory",
Connections: 5, Connections: 5,
}}, }},
{"b2:foobar", Config{ {S: "b2:foobar", Cfg: Config{
Bucket: "foobar", Bucket: "foobar",
Prefix: "", Prefix: "",
Connections: 5, Connections: 5,
}}, }},
{"b2:foobar:", Config{ {S: "b2:foobar:", Cfg: Config{
Bucket: "foobar", Bucket: "foobar",
Prefix: "", Prefix: "",
Connections: 5, Connections: 5,
}}, }},
{"b2:foobar:/", Config{ {S: "b2:foobar:/", Cfg: Config{
Bucket: "foobar", Bucket: "foobar",
Prefix: "", Prefix: "",
Connections: 5, Connections: 5,
@ -39,19 +40,7 @@ var configTests = []struct {
} }
func TestParseConfig(t *testing.T) { func TestParseConfig(t *testing.T) {
for _, test := range configTests { test.ParseConfigTester(t, ParseConfig, configTests)
t.Run("", func(t *testing.T) {
cfg, err := ParseConfig(test.s)
if err != nil {
t.Fatalf("%s failed: %v", test.s, err)
}
if cfg != test.cfg {
t.Fatalf("input: %s\n wrong config, want:\n %#v\ngot:\n %#v",
test.s, test.cfg, cfg)
}
})
}
} }
var invalidConfigTests = []struct { var invalidConfigTests = []struct {

View file

@ -1,11 +1,13 @@
package gs package gs
import ( import (
"os"
"path" "path"
"strings" "strings"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options" "github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
) )
// Config contains all configuration necessary to connect to a Google Cloud Storage // Config contains all configuration necessary to connect to a Google Cloud Storage
@ -34,7 +36,7 @@ func init() {
// ParseConfig parses the string s and extracts the gcs config. The // ParseConfig parses the string s and extracts the gcs config. The
// supported configuration format is gs:bucketName:/[prefix]. // supported configuration format is gs:bucketName:/[prefix].
func ParseConfig(s string) (interface{}, error) { func ParseConfig(s string) (*Config, error) {
if !strings.HasPrefix(s, "gs:") { if !strings.HasPrefix(s, "gs:") {
return nil, errors.New("gs: invalid format") return nil, errors.New("gs: invalid format")
} }
@ -54,5 +56,15 @@ func ParseConfig(s string) (interface{}, error) {
cfg := NewConfig() cfg := NewConfig()
cfg.Bucket = bucket cfg.Bucket = bucket
cfg.Prefix = prefix cfg.Prefix = prefix
return cfg, nil return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) error {
if cfg.ProjectID == "" {
cfg.ProjectID = os.Getenv(prefix + "GOOGLE_PROJECT_ID")
}
return nil
} }

View file

@ -1,24 +1,25 @@
package gs package gs
import "testing" import (
"testing"
var configTests = []struct { "github.com/restic/restic/internal/backend/test"
s string )
cfg Config
}{ var configTests = []test.ConfigTestData[Config]{
{"gs:bucketname:/", Config{ {S: "gs:bucketname:/", Cfg: Config{
Bucket: "bucketname", Bucket: "bucketname",
Prefix: "", Prefix: "",
Connections: 5, Connections: 5,
Region: "us", Region: "us",
}}, }},
{"gs:bucketname:/prefix/directory", Config{ {S: "gs:bucketname:/prefix/directory", Cfg: Config{
Bucket: "bucketname", Bucket: "bucketname",
Prefix: "prefix/directory", Prefix: "prefix/directory",
Connections: 5, Connections: 5,
Region: "us", Region: "us",
}}, }},
{"gs:bucketname:/prefix/directory/", Config{ {S: "gs:bucketname:/prefix/directory/", Cfg: Config{
Bucket: "bucketname", Bucket: "bucketname",
Prefix: "prefix/directory", Prefix: "prefix/directory",
Connections: 5, Connections: 5,
@ -27,17 +28,5 @@ var configTests = []struct {
} }
func TestParseConfig(t *testing.T) { func TestParseConfig(t *testing.T) {
for i, test := range configTests { test.ParseConfigTester(t, ParseConfig, configTests)
cfg, err := ParseConfig(test.s)
if err != nil {
t.Errorf("test %d:%s failed: %v", i, test.s, err)
continue
}
if cfg != test.cfg {
t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v",
i, test.s, test.cfg, cfg)
continue
}
}
} }

View file

@ -15,33 +15,30 @@ import (
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
func newGSTestSuite(t testing.TB) *test.Suite { func newGSTestSuite(t testing.TB) *test.Suite[gs.Config] {
tr, err := backend.Transport(backend.TransportOptions{}) tr, err := backend.Transport(backend.TransportOptions{})
if err != nil { if err != nil {
t.Fatalf("cannot create transport for tests: %v", err) t.Fatalf("cannot create transport for tests: %v", err)
} }
return &test.Suite{ return &test.Suite[gs.Config]{
// do not use excessive data // do not use excessive data
MinimalData: true, MinimalData: true,
// NewConfig returns a config for a new temporary backend that will be used in tests. // NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) { NewConfig: func() (*gs.Config, error) {
gscfg, err := gs.ParseConfig(os.Getenv("RESTIC_TEST_GS_REPOSITORY")) cfg, err := gs.ParseConfig(os.Getenv("RESTIC_TEST_GS_REPOSITORY"))
if err != nil { if err != nil {
return nil, err return nil, err
} }
cfg := gscfg.(gs.Config)
cfg.ProjectID = os.Getenv("RESTIC_TEST_GS_PROJECT_ID") cfg.ProjectID = os.Getenv("RESTIC_TEST_GS_PROJECT_ID")
cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
return cfg, nil return cfg, nil
}, },
// CreateFn is a function that creates a temporary repository for the tests. // CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) { Create: func(cfg gs.Config) (restic.Backend, error) {
cfg := config.(gs.Config)
be, err := gs.Create(context.Background(), cfg, tr) be, err := gs.Create(context.Background(), cfg, tr)
if err != nil { if err != nil {
return nil, err return nil, err
@ -60,15 +57,12 @@ func newGSTestSuite(t testing.TB) *test.Suite {
}, },
// OpenFn is a function that opens a previously created temporary repository. // OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) { Open: func(cfg gs.Config) (restic.Backend, error) {
cfg := config.(gs.Config)
return gs.Open(cfg, tr) return gs.Open(cfg, tr)
}, },
// CleanupFn removes data created during the tests. // CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error { Cleanup: func(cfg gs.Config) error {
cfg := config.(gs.Config)
be, err := gs.Open(cfg, tr) be, err := gs.Open(cfg, tr)
if err != nil { if err != nil {
return err return err

View file

@ -27,12 +27,12 @@ func init() {
} }
// ParseConfig parses a local backend config. // ParseConfig parses a local backend config.
func ParseConfig(s string) (interface{}, error) { func ParseConfig(s string) (*Config, error) {
if !strings.HasPrefix(s, "local:") { if !strings.HasPrefix(s, "local:") {
return nil, errors.New(`invalid format, prefix "local" not found`) return nil, errors.New(`invalid format, prefix "local" not found`)
} }
cfg := NewConfig() cfg := NewConfig()
cfg.Path = s[6:] cfg.Path = s[6:]
return cfg, nil return &cfg, nil
} }

View file

@ -0,0 +1,18 @@
package local
import (
"testing"
"github.com/restic/restic/internal/backend/test"
)
var configTests = []test.ConfigTestData[Config]{
{S: "local:/some/path", Cfg: Config{
Path: "/some/path",
Connections: 2,
}},
}
func TestParseConfig(t *testing.T) {
test.ParseConfigTester(t, ParseConfig, configTests)
}

View file

@ -148,6 +148,13 @@ func (b *Local) Save(_ context.Context, h restic.Handle, rd restic.RewindReader)
} }
}(f) }(f)
// preallocate disk space
if size := rd.Length(); size > 0 {
if err := fs.PreallocateFile(f, size); err != nil {
debug.Log("Failed to preallocate %v with size %v: %v", finalname, size, err)
}
}
// save data, then sync // save data, then sync
wbytes, err := io.Copy(f, rd) wbytes, err := io.Copy(f, rd)
if err != nil { if err != nil {

View file

@ -12,10 +12,10 @@ import (
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
func newTestSuite(t testing.TB) *test.Suite { func newTestSuite(t testing.TB) *test.Suite[local.Config] {
return &test.Suite{ return &test.Suite[local.Config]{
// NewConfig returns a config for a new temporary backend that will be used in tests. // NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) { NewConfig: func() (*local.Config, error) {
dir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-local-") dir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-local-")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -23,7 +23,7 @@ func newTestSuite(t testing.TB) *test.Suite {
t.Logf("create new backend at %v", dir) t.Logf("create new backend at %v", dir)
cfg := local.Config{ cfg := &local.Config{
Path: dir, Path: dir,
Connections: 2, Connections: 2,
} }
@ -31,20 +31,17 @@ func newTestSuite(t testing.TB) *test.Suite {
}, },
// CreateFn is a function that creates a temporary repository for the tests. // CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) { Create: func(cfg local.Config) (restic.Backend, error) {
cfg := config.(local.Config)
return local.Create(context.TODO(), cfg) return local.Create(context.TODO(), cfg)
}, },
// OpenFn is a function that opens a previously created temporary repository. // OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) { Open: func(cfg local.Config) (restic.Backend, error) {
cfg := config.(local.Config)
return local.Open(context.TODO(), cfg) return local.Open(context.TODO(), cfg)
}, },
// CleanupFn removes data created during the tests. // CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error { Cleanup: func(cfg local.Config) error {
cfg := config.(local.Config)
if !rtest.TestCleanupTempDirs { if !rtest.TestCleanupTempDirs {
t.Logf("leaving test backend dir at %v", cfg.Path) t.Logf("leaving test backend dir at %v", cfg.Path)
} }

View file

@ -30,19 +30,25 @@ type parser struct {
stripPassword func(string) string stripPassword func(string) string
} }
func configToAny[C any](parser func(string) (*C, error)) func(string) (interface{}, error) {
return func(s string) (interface{}, error) {
return parser(s)
}
}
// parsers is a list of valid config parsers for the backends. The first parser // parsers is a list of valid config parsers for the backends. The first parser
// is the fallback and should always be set to the local backend. // is the fallback and should always be set to the local backend.
var parsers = []parser{ var parsers = []parser{
{"b2", b2.ParseConfig, noPassword}, {"b2", configToAny(b2.ParseConfig), noPassword},
{"local", local.ParseConfig, noPassword}, {"local", configToAny(local.ParseConfig), noPassword},
{"sftp", sftp.ParseConfig, noPassword}, {"sftp", configToAny(sftp.ParseConfig), noPassword},
{"s3", s3.ParseConfig, noPassword}, {"s3", configToAny(s3.ParseConfig), noPassword},
{"gs", gs.ParseConfig, noPassword}, {"gs", configToAny(gs.ParseConfig), noPassword},
{"azure", azure.ParseConfig, noPassword}, {"azure", configToAny(azure.ParseConfig), noPassword},
{"swift", swift.ParseConfig, noPassword}, {"swift", configToAny(swift.ParseConfig), noPassword},
{"rest", rest.ParseConfig, rest.StripPassword}, {"rest", configToAny(rest.ParseConfig), rest.StripPassword},
{"rclone", rclone.ParseConfig, noPassword}, {"rclone", configToAny(rclone.ParseConfig), noPassword},
{"smb", smb.ParseConfig, noPassword}, {"smb", configToAny(smb.ParseConfig), noPassword},
} }
// noPassword returns the repository location unchanged (there's no sensitive information there) // noPassword returns the repository location unchanged (there's no sensitive information there)

View file

@ -29,7 +29,7 @@ var parseTests = []struct {
{ {
"local:/srv/repo", "local:/srv/repo",
Location{Scheme: "local", Location{Scheme: "local",
Config: local.Config{ Config: &local.Config{
Path: "/srv/repo", Path: "/srv/repo",
Connections: 2, Connections: 2,
}, },
@ -38,7 +38,7 @@ var parseTests = []struct {
{ {
"local:dir1/dir2", "local:dir1/dir2",
Location{Scheme: "local", Location{Scheme: "local",
Config: local.Config{ Config: &local.Config{
Path: "dir1/dir2", Path: "dir1/dir2",
Connections: 2, Connections: 2,
}, },
@ -47,7 +47,7 @@ var parseTests = []struct {
{ {
"local:dir1/dir2", "local:dir1/dir2",
Location{Scheme: "local", Location{Scheme: "local",
Config: local.Config{ Config: &local.Config{
Path: "dir1/dir2", Path: "dir1/dir2",
Connections: 2, Connections: 2,
}, },
@ -56,7 +56,7 @@ var parseTests = []struct {
{ {
"dir1/dir2", "dir1/dir2",
Location{Scheme: "local", Location{Scheme: "local",
Config: local.Config{ Config: &local.Config{
Path: "dir1/dir2", Path: "dir1/dir2",
Connections: 2, Connections: 2,
}, },
@ -65,7 +65,7 @@ var parseTests = []struct {
{ {
"/dir1/dir2", "/dir1/dir2",
Location{Scheme: "local", Location{Scheme: "local",
Config: local.Config{ Config: &local.Config{
Path: "/dir1/dir2", Path: "/dir1/dir2",
Connections: 2, Connections: 2,
}, },
@ -74,7 +74,7 @@ var parseTests = []struct {
{ {
"local:../dir1/dir2", "local:../dir1/dir2",
Location{Scheme: "local", Location{Scheme: "local",
Config: local.Config{ Config: &local.Config{
Path: "../dir1/dir2", Path: "../dir1/dir2",
Connections: 2, Connections: 2,
}, },
@ -83,7 +83,7 @@ var parseTests = []struct {
{ {
"/dir1/dir2", "/dir1/dir2",
Location{Scheme: "local", Location{Scheme: "local",
Config: local.Config{ Config: &local.Config{
Path: "/dir1/dir2", Path: "/dir1/dir2",
Connections: 2, Connections: 2,
}, },
@ -92,7 +92,7 @@ var parseTests = []struct {
{ {
"/dir1:foobar/dir2", "/dir1:foobar/dir2",
Location{Scheme: "local", Location{Scheme: "local",
Config: local.Config{ Config: &local.Config{
Path: "/dir1:foobar/dir2", Path: "/dir1:foobar/dir2",
Connections: 2, Connections: 2,
}, },
@ -101,7 +101,7 @@ var parseTests = []struct {
{ {
`\dir1\foobar\dir2`, `\dir1\foobar\dir2`,
Location{Scheme: "local", Location{Scheme: "local",
Config: local.Config{ Config: &local.Config{
Path: `\dir1\foobar\dir2`, Path: `\dir1\foobar\dir2`,
Connections: 2, Connections: 2,
}, },
@ -110,7 +110,7 @@ var parseTests = []struct {
{ {
`c:\dir1\foobar\dir2`, `c:\dir1\foobar\dir2`,
Location{Scheme: "local", Location{Scheme: "local",
Config: local.Config{ Config: &local.Config{
Path: `c:\dir1\foobar\dir2`, Path: `c:\dir1\foobar\dir2`,
Connections: 2, Connections: 2,
}, },
@ -119,7 +119,7 @@ var parseTests = []struct {
{ {
`C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`, `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`,
Location{Scheme: "local", Location{Scheme: "local",
Config: local.Config{ Config: &local.Config{
Path: `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`, Path: `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`,
Connections: 2, Connections: 2,
}, },
@ -128,7 +128,7 @@ var parseTests = []struct {
{ {
`c:/dir1/foobar/dir2`, `c:/dir1/foobar/dir2`,
Location{Scheme: "local", Location{Scheme: "local",
Config: local.Config{ Config: &local.Config{
Path: `c:/dir1/foobar/dir2`, Path: `c:/dir1/foobar/dir2`,
Connections: 2, Connections: 2,
}, },
@ -137,7 +137,7 @@ var parseTests = []struct {
{ {
"sftp:user@host:/srv/repo", "sftp:user@host:/srv/repo",
Location{Scheme: "sftp", Location{Scheme: "sftp",
Config: sftp.Config{ Config: &sftp.Config{
User: "user", User: "user",
Host: "host", Host: "host",
Path: "/srv/repo", Path: "/srv/repo",
@ -148,7 +148,7 @@ var parseTests = []struct {
{ {
"sftp:host:/srv/repo", "sftp:host:/srv/repo",
Location{Scheme: "sftp", Location{Scheme: "sftp",
Config: sftp.Config{ Config: &sftp.Config{
User: "", User: "",
Host: "host", Host: "host",
Path: "/srv/repo", Path: "/srv/repo",
@ -159,7 +159,7 @@ var parseTests = []struct {
{ {
"sftp://user@host/srv/repo", "sftp://user@host/srv/repo",
Location{Scheme: "sftp", Location{Scheme: "sftp",
Config: sftp.Config{ Config: &sftp.Config{
User: "user", User: "user",
Host: "host", Host: "host",
Path: "srv/repo", Path: "srv/repo",
@ -170,7 +170,7 @@ var parseTests = []struct {
{ {
"sftp://user@host//srv/repo", "sftp://user@host//srv/repo",
Location{Scheme: "sftp", Location{Scheme: "sftp",
Config: sftp.Config{ Config: &sftp.Config{
User: "user", User: "user",
Host: "host", Host: "host",
Path: "/srv/repo", Path: "/srv/repo",
@ -182,7 +182,7 @@ var parseTests = []struct {
{ {
"s3://eu-central-1/bucketname", "s3://eu-central-1/bucketname",
Location{Scheme: "s3", Location{Scheme: "s3",
Config: s3.Config{ Config: &s3.Config{
Endpoint: "eu-central-1", Endpoint: "eu-central-1",
Bucket: "bucketname", Bucket: "bucketname",
Prefix: "", Prefix: "",
@ -193,7 +193,7 @@ var parseTests = []struct {
{ {
"s3://hostname.foo/bucketname", "s3://hostname.foo/bucketname",
Location{Scheme: "s3", Location{Scheme: "s3",
Config: s3.Config{ Config: &s3.Config{
Endpoint: "hostname.foo", Endpoint: "hostname.foo",
Bucket: "bucketname", Bucket: "bucketname",
Prefix: "", Prefix: "",
@ -204,7 +204,7 @@ var parseTests = []struct {
{ {
"s3://hostname.foo/bucketname/prefix/directory", "s3://hostname.foo/bucketname/prefix/directory",
Location{Scheme: "s3", Location{Scheme: "s3",
Config: s3.Config{ Config: &s3.Config{
Endpoint: "hostname.foo", Endpoint: "hostname.foo",
Bucket: "bucketname", Bucket: "bucketname",
Prefix: "prefix/directory", Prefix: "prefix/directory",
@ -215,7 +215,7 @@ var parseTests = []struct {
{ {
"s3:eu-central-1/repo", "s3:eu-central-1/repo",
Location{Scheme: "s3", Location{Scheme: "s3",
Config: s3.Config{ Config: &s3.Config{
Endpoint: "eu-central-1", Endpoint: "eu-central-1",
Bucket: "repo", Bucket: "repo",
Prefix: "", Prefix: "",
@ -226,7 +226,7 @@ var parseTests = []struct {
{ {
"s3:eu-central-1/repo/prefix/directory", "s3:eu-central-1/repo/prefix/directory",
Location{Scheme: "s3", Location{Scheme: "s3",
Config: s3.Config{ Config: &s3.Config{
Endpoint: "eu-central-1", Endpoint: "eu-central-1",
Bucket: "repo", Bucket: "repo",
Prefix: "prefix/directory", Prefix: "prefix/directory",
@ -237,7 +237,7 @@ var parseTests = []struct {
{ {
"s3:https://hostname.foo/repo", "s3:https://hostname.foo/repo",
Location{Scheme: "s3", Location{Scheme: "s3",
Config: s3.Config{ Config: &s3.Config{
Endpoint: "hostname.foo", Endpoint: "hostname.foo",
Bucket: "repo", Bucket: "repo",
Prefix: "", Prefix: "",
@ -248,7 +248,7 @@ var parseTests = []struct {
{ {
"s3:https://hostname.foo/repo/prefix/directory", "s3:https://hostname.foo/repo/prefix/directory",
Location{Scheme: "s3", Location{Scheme: "s3",
Config: s3.Config{ Config: &s3.Config{
Endpoint: "hostname.foo", Endpoint: "hostname.foo",
Bucket: "repo", Bucket: "repo",
Prefix: "prefix/directory", Prefix: "prefix/directory",
@ -259,7 +259,7 @@ var parseTests = []struct {
{ {
"s3:http://hostname.foo/repo", "s3:http://hostname.foo/repo",
Location{Scheme: "s3", Location{Scheme: "s3",
Config: s3.Config{ Config: &s3.Config{
Endpoint: "hostname.foo", Endpoint: "hostname.foo",
Bucket: "repo", Bucket: "repo",
Prefix: "", Prefix: "",
@ -271,7 +271,7 @@ var parseTests = []struct {
{ {
"swift:container17:/", "swift:container17:/",
Location{Scheme: "swift", Location{Scheme: "swift",
Config: swift.Config{ Config: &swift.Config{
Container: "container17", Container: "container17",
Prefix: "", Prefix: "",
Connections: 5, Connections: 5,
@ -281,7 +281,7 @@ var parseTests = []struct {
{ {
"swift:container17:/prefix97", "swift:container17:/prefix97",
Location{Scheme: "swift", Location{Scheme: "swift",
Config: swift.Config{ Config: &swift.Config{
Container: "container17", Container: "container17",
Prefix: "prefix97", Prefix: "prefix97",
Connections: 5, Connections: 5,
@ -291,7 +291,7 @@ var parseTests = []struct {
{ {
"rest:http://hostname.foo:1234/", "rest:http://hostname.foo:1234/",
Location{Scheme: "rest", Location{Scheme: "rest",
Config: rest.Config{ Config: &rest.Config{
URL: parseURL("http://hostname.foo:1234/"), URL: parseURL("http://hostname.foo:1234/"),
Connections: 5, Connections: 5,
}, },
@ -299,7 +299,7 @@ var parseTests = []struct {
}, },
{ {
"b2:bucketname:/prefix", Location{Scheme: "b2", "b2:bucketname:/prefix", Location{Scheme: "b2",
Config: b2.Config{ Config: &b2.Config{
Bucket: "bucketname", Bucket: "bucketname",
Prefix: "prefix", Prefix: "prefix",
Connections: 5, Connections: 5,
@ -308,7 +308,7 @@ var parseTests = []struct {
}, },
{ {
"b2:bucketname", Location{Scheme: "b2", "b2:bucketname", Location{Scheme: "b2",
Config: b2.Config{ Config: &b2.Config{
Bucket: "bucketname", Bucket: "bucketname",
Prefix: "", Prefix: "",
Connections: 5, Connections: 5,

View file

@ -15,19 +15,19 @@ type memConfig struct {
be restic.Backend be restic.Backend
} }
func newTestSuite() *test.Suite { func newTestSuite() *test.Suite[*memConfig] {
return &test.Suite{ return &test.Suite[*memConfig]{
// NewConfig returns a config for a new temporary backend that will be used in tests. // NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) { NewConfig: func() (**memConfig, error) {
return &memConfig{}, nil cfg := &memConfig{}
return &cfg, nil
}, },
// CreateFn is a function that creates a temporary repository for the tests. // CreateFn is a function that creates a temporary repository for the tests.
Create: func(cfg interface{}) (restic.Backend, error) { Create: func(cfg *memConfig) (restic.Backend, error) {
c := cfg.(*memConfig) if cfg.be != nil {
if c.be != nil { _, err := cfg.be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
_, err := c.be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile}) if err != nil && !cfg.be.IsNotExist(err) {
if err != nil && !c.be.IsNotExist(err) {
return nil, err return nil, err
} }
@ -36,21 +36,20 @@ func newTestSuite() *test.Suite {
} }
} }
c.be = mem.New() cfg.be = mem.New()
return c.be, nil return cfg.be, nil
}, },
// OpenFn is a function that opens a previously created temporary repository. // OpenFn is a function that opens a previously created temporary repository.
Open: func(cfg interface{}) (restic.Backend, error) { Open: func(cfg *memConfig) (restic.Backend, error) {
c := cfg.(*memConfig) if cfg.be == nil {
if c.be == nil { cfg.be = mem.New()
c.be = mem.New()
} }
return c.be, nil return cfg.be, nil
}, },
// CleanupFn removes data created during the tests. // CleanupFn removes data created during the tests.
Cleanup: func(cfg interface{}) error { Cleanup: func(cfg *memConfig) error {
// no cleanup needed // no cleanup needed
return nil return nil
}, },

View file

@ -12,22 +12,21 @@ import (
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
func newTestSuite(t testing.TB) *test.Suite { func newTestSuite(t testing.TB) *test.Suite[rclone.Config] {
dir := rtest.TempDir(t) dir := rtest.TempDir(t)
return &test.Suite{ return &test.Suite[rclone.Config]{
// NewConfig returns a config for a new temporary backend that will be used in tests. // NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) { NewConfig: func() (*rclone.Config, error) {
t.Logf("use backend at %v", dir) t.Logf("use backend at %v", dir)
cfg := rclone.NewConfig() cfg := rclone.NewConfig()
cfg.Remote = dir cfg.Remote = dir
return cfg, nil return &cfg, nil
}, },
// CreateFn is a function that creates a temporary repository for the tests. // CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) { Create: func(cfg rclone.Config) (restic.Backend, error) {
t.Logf("Create()") t.Logf("Create()")
cfg := config.(rclone.Config)
be, err := rclone.Create(context.TODO(), cfg) be, err := rclone.Create(context.TODO(), cfg)
var e *exec.Error var e *exec.Error
if errors.As(err, &e) && e.Err == exec.ErrNotFound { if errors.As(err, &e) && e.Err == exec.ErrNotFound {
@ -38,9 +37,8 @@ func newTestSuite(t testing.TB) *test.Suite {
}, },
// OpenFn is a function that opens a previously created temporary repository. // OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) { Open: func(cfg rclone.Config) (restic.Backend, error) {
t.Logf("Open()") t.Logf("Open()")
cfg := config.(rclone.Config)
return rclone.Open(cfg, nil) return rclone.Open(cfg, nil)
}, },
} }

View file

@ -34,7 +34,7 @@ func NewConfig() Config {
} }
// ParseConfig parses the string s and extracts the remote server URL. // ParseConfig parses the string s and extracts the remote server URL.
func ParseConfig(s string) (interface{}, error) { func ParseConfig(s string) (*Config, error) {
if !strings.HasPrefix(s, "rclone:") { if !strings.HasPrefix(s, "rclone:") {
return nil, errors.New("invalid rclone backend specification") return nil, errors.New("invalid rclone backend specification")
} }
@ -42,5 +42,5 @@ func ParseConfig(s string) (interface{}, error) {
s = s[7:] s = s[7:]
cfg := NewConfig() cfg := NewConfig()
cfg.Remote = s cfg.Remote = s
return cfg, nil return &cfg, nil
} }

View file

@ -1,37 +1,24 @@
package rclone package rclone
import ( import (
"reflect"
"testing" "testing"
"github.com/restic/restic/internal/backend/test"
) )
func TestParseConfig(t *testing.T) { var configTests = []test.ConfigTestData[Config]{
var tests = []struct { {
s string S: "rclone:local:foo:/bar",
cfg Config Cfg: Config{
}{ Remote: "local:foo:/bar",
{ Program: defaultConfig.Program,
"rclone:local:foo:/bar", Args: defaultConfig.Args,
Config{ Connections: defaultConfig.Connections,
Remote: "local:foo:/bar", Timeout: defaultConfig.Timeout,
Program: defaultConfig.Program,
Args: defaultConfig.Args,
Connections: defaultConfig.Connections,
Timeout: defaultConfig.Timeout,
},
}, },
} },
}
for _, test := range tests {
t.Run("", func(t *testing.T) { func TestParseConfig(t *testing.T) {
cfg, err := ParseConfig(test.s) test.ParseConfigTester(t, ParseConfig, configTests)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(cfg, test.cfg) {
t.Fatalf("wrong config, want:\n %v\ngot:\n %v", test.cfg, cfg)
}
})
}
} }

View file

@ -26,7 +26,7 @@ func NewConfig() Config {
} }
// ParseConfig parses the string s and extracts the REST server URL. // ParseConfig parses the string s and extracts the REST server URL.
func ParseConfig(s string) (interface{}, error) { func ParseConfig(s string) (*Config, error) {
if !strings.HasPrefix(s, "rest:") { if !strings.HasPrefix(s, "rest:") {
return nil, errors.New("invalid REST backend specification") return nil, errors.New("invalid REST backend specification")
} }
@ -40,7 +40,7 @@ func ParseConfig(s string) (interface{}, error) {
cfg := NewConfig() cfg := NewConfig()
cfg.URL = u cfg.URL = u
return cfg, nil return &cfg, nil
} }
// StripPassword removes the password from the URL // StripPassword removes the password from the URL

View file

@ -2,8 +2,9 @@ package rest
import ( import (
"net/url" "net/url"
"reflect"
"testing" "testing"
"github.com/restic/restic/internal/backend/test"
) )
func parseURL(s string) *url.URL { func parseURL(s string) *url.URL {
@ -15,20 +16,17 @@ func parseURL(s string) *url.URL {
return u return u
} }
var configTests = []struct { var configTests = []test.ConfigTestData[Config]{
s string
cfg Config
}{
{ {
s: "rest:http://localhost:1234", S: "rest:http://localhost:1234",
cfg: Config{ Cfg: Config{
URL: parseURL("http://localhost:1234/"), URL: parseURL("http://localhost:1234/"),
Connections: 5, Connections: 5,
}, },
}, },
{ {
s: "rest:http://localhost:1234/", S: "rest:http://localhost:1234/",
cfg: Config{ Cfg: Config{
URL: parseURL("http://localhost:1234/"), URL: parseURL("http://localhost:1234/"),
Connections: 5, Connections: 5,
}, },
@ -36,17 +34,5 @@ var configTests = []struct {
} }
func TestParseConfig(t *testing.T) { func TestParseConfig(t *testing.T) {
for _, test := range configTests { test.ParseConfigTester(t, ParseConfig, configTests)
t.Run("", func(t *testing.T) {
cfg, err := ParseConfig(test.s)
if err != nil {
t.Fatalf("%s failed: %v", test.s, err)
}
if !reflect.DeepEqual(cfg, test.cfg) {
t.Fatalf("\ninput: %s\n wrong config, want:\n %v\ngot:\n %v",
test.s, test.cfg, cfg)
}
})
}
} }

View file

@ -67,36 +67,34 @@ func runRESTServer(ctx context.Context, t testing.TB, dir string) (*url.URL, fun
return url, cleanup return url, cleanup
} }
func newTestSuite(_ context.Context, t testing.TB, url *url.URL, minimalData bool) *test.Suite { func newTestSuite(_ context.Context, t testing.TB, url *url.URL, minimalData bool) *test.Suite[rest.Config] {
tr, err := backend.Transport(backend.TransportOptions{}) tr, err := backend.Transport(backend.TransportOptions{})
if err != nil { if err != nil {
t.Fatalf("cannot create transport for tests: %v", err) t.Fatalf("cannot create transport for tests: %v", err)
} }
return &test.Suite{ return &test.Suite[rest.Config]{
MinimalData: minimalData, MinimalData: minimalData,
// NewConfig returns a config for a new temporary backend that will be used in tests. // NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) { NewConfig: func() (*rest.Config, error) {
cfg := rest.NewConfig() cfg := rest.NewConfig()
cfg.URL = url cfg.URL = url
return cfg, nil return &cfg, nil
}, },
// CreateFn is a function that creates a temporary repository for the tests. // CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) { Create: func(cfg rest.Config) (restic.Backend, error) {
cfg := config.(rest.Config)
return rest.Create(context.TODO(), cfg, tr) return rest.Create(context.TODO(), cfg, tr)
}, },
// OpenFn is a function that opens a previously created temporary repository. // OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) { Open: func(cfg rest.Config) (restic.Backend, error) {
cfg := config.(rest.Config)
return rest.Open(cfg, tr) return rest.Open(cfg, tr)
}, },
// CleanupFn removes data created during the tests. // CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error { Cleanup: func(cfg rest.Config) error {
return nil return nil
}, },
} }
@ -130,12 +128,10 @@ func TestBackendRESTExternalServer(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
c := cfg.(rest.Config)
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()
newTestSuite(ctx, t, c.URL, true).RunTests(t) newTestSuite(ctx, t, cfg.URL, true).RunTests(t)
} }
func BenchmarkBackendREST(t *testing.B) { func BenchmarkBackendREST(t *testing.B) {

View file

@ -2,11 +2,13 @@ package s3
import ( import (
"net/url" "net/url"
"os"
"path" "path"
"strings" "strings"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options" "github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
) )
// Config contains all configuration necessary to connect to an s3 compatible // Config contains all configuration necessary to connect to an s3 compatible
@ -44,7 +46,7 @@ func init() {
// supported configuration formats are s3://host/bucketname/prefix and // supported configuration formats are s3://host/bucketname/prefix and
// s3:host/bucketname/prefix. The host can also be a valid s3 region // s3:host/bucketname/prefix. The host can also be a valid s3 region
// name. If no prefix is given the prefix "restic" will be used. // name. If no prefix is given the prefix "restic" will be used.
func ParseConfig(s string) (interface{}, error) { func ParseConfig(s string) (*Config, error) {
switch { switch {
case strings.HasPrefix(s, "s3:http"): case strings.HasPrefix(s, "s3:http"):
// assume that a URL has been specified, parse it and // assume that a URL has been specified, parse it and
@ -75,7 +77,7 @@ func ParseConfig(s string) (interface{}, error) {
return createConfig(endpoint, bucket, prefix, false) return createConfig(endpoint, bucket, prefix, false)
} }
func createConfig(endpoint, bucket, prefix string, useHTTP bool) (interface{}, error) { func createConfig(endpoint, bucket, prefix string, useHTTP bool) (*Config, error) {
if endpoint == "" { if endpoint == "" {
return nil, errors.New("s3: invalid format, host/region or bucket name not found") return nil, errors.New("s3: invalid format, host/region or bucket name not found")
} }
@ -89,5 +91,30 @@ func createConfig(endpoint, bucket, prefix string, useHTTP bool) (interface{}, e
cfg.UseHTTP = useHTTP cfg.UseHTTP = useHTTP
cfg.Bucket = bucket cfg.Bucket = bucket
cfg.Prefix = prefix cfg.Prefix = prefix
return cfg, nil return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) error {
if cfg.KeyID == "" {
cfg.KeyID = os.Getenv(prefix + "AWS_ACCESS_KEY_ID")
}
if cfg.Secret.String() == "" {
cfg.Secret = options.NewSecretString(os.Getenv(prefix + "AWS_SECRET_ACCESS_KEY"))
}
if cfg.KeyID == "" && cfg.Secret.String() != "" {
return errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty")
} else if cfg.KeyID != "" && cfg.Secret.String() == "" {
return errors.Fatalf("unable to open S3 backend: Secret ($AWS_SECRET_ACCESS_KEY) is empty")
}
if cfg.Region == "" {
cfg.Region = os.Getenv(prefix + "AWS_DEFAULT_REGION")
}
return nil
} }

View file

@ -3,94 +3,93 @@ package s3
import ( import (
"strings" "strings"
"testing" "testing"
"github.com/restic/restic/internal/backend/test"
) )
var configTests = []struct { var configTests = []test.ConfigTestData[Config]{
s string {S: "s3://eu-central-1/bucketname", Cfg: Config{
cfg Config
}{
{"s3://eu-central-1/bucketname", Config{
Endpoint: "eu-central-1", Endpoint: "eu-central-1",
Bucket: "bucketname", Bucket: "bucketname",
Prefix: "", Prefix: "",
Connections: 5, Connections: 5,
}}, }},
{"s3://eu-central-1/bucketname/", Config{ {S: "s3://eu-central-1/bucketname/", Cfg: Config{
Endpoint: "eu-central-1", Endpoint: "eu-central-1",
Bucket: "bucketname", Bucket: "bucketname",
Prefix: "", Prefix: "",
Connections: 5, Connections: 5,
}}, }},
{"s3://eu-central-1/bucketname/prefix/directory", Config{ {S: "s3://eu-central-1/bucketname/prefix/directory", Cfg: Config{
Endpoint: "eu-central-1", Endpoint: "eu-central-1",
Bucket: "bucketname", Bucket: "bucketname",
Prefix: "prefix/directory", Prefix: "prefix/directory",
Connections: 5, Connections: 5,
}}, }},
{"s3://eu-central-1/bucketname/prefix/directory/", Config{ {S: "s3://eu-central-1/bucketname/prefix/directory/", Cfg: Config{
Endpoint: "eu-central-1", Endpoint: "eu-central-1",
Bucket: "bucketname", Bucket: "bucketname",
Prefix: "prefix/directory", Prefix: "prefix/directory",
Connections: 5, Connections: 5,
}}, }},
{"s3:eu-central-1/foobar", Config{ {S: "s3:eu-central-1/foobar", Cfg: Config{
Endpoint: "eu-central-1", Endpoint: "eu-central-1",
Bucket: "foobar", Bucket: "foobar",
Prefix: "", Prefix: "",
Connections: 5, Connections: 5,
}}, }},
{"s3:eu-central-1/foobar/", Config{ {S: "s3:eu-central-1/foobar/", Cfg: Config{
Endpoint: "eu-central-1", Endpoint: "eu-central-1",
Bucket: "foobar", Bucket: "foobar",
Prefix: "", Prefix: "",
Connections: 5, Connections: 5,
}}, }},
{"s3:eu-central-1/foobar/prefix/directory", Config{ {S: "s3:eu-central-1/foobar/prefix/directory", Cfg: Config{
Endpoint: "eu-central-1", Endpoint: "eu-central-1",
Bucket: "foobar", Bucket: "foobar",
Prefix: "prefix/directory", Prefix: "prefix/directory",
Connections: 5, Connections: 5,
}}, }},
{"s3:eu-central-1/foobar/prefix/directory/", Config{ {S: "s3:eu-central-1/foobar/prefix/directory/", Cfg: Config{
Endpoint: "eu-central-1", Endpoint: "eu-central-1",
Bucket: "foobar", Bucket: "foobar",
Prefix: "prefix/directory", Prefix: "prefix/directory",
Connections: 5, Connections: 5,
}}, }},
{"s3:https://hostname:9999/foobar", Config{ {S: "s3:https://hostname:9999/foobar", Cfg: Config{
Endpoint: "hostname:9999", Endpoint: "hostname:9999",
Bucket: "foobar", Bucket: "foobar",
Prefix: "", Prefix: "",
Connections: 5, Connections: 5,
}}, }},
{"s3:https://hostname:9999/foobar/", Config{ {S: "s3:https://hostname:9999/foobar/", Cfg: Config{
Endpoint: "hostname:9999", Endpoint: "hostname:9999",
Bucket: "foobar", Bucket: "foobar",
Prefix: "", Prefix: "",
Connections: 5, Connections: 5,
}}, }},
{"s3:http://hostname:9999/foobar", Config{ {S: "s3:http://hostname:9999/foobar", Cfg: Config{
Endpoint: "hostname:9999", Endpoint: "hostname:9999",
Bucket: "foobar", Bucket: "foobar",
Prefix: "", Prefix: "",
UseHTTP: true, UseHTTP: true,
Connections: 5, Connections: 5,
}}, }},
{"s3:http://hostname:9999/foobar/", Config{ {S: "s3:http://hostname:9999/foobar/", Cfg: Config{
Endpoint: "hostname:9999", Endpoint: "hostname:9999",
Bucket: "foobar", Bucket: "foobar",
Prefix: "", Prefix: "",
UseHTTP: true, UseHTTP: true,
Connections: 5, Connections: 5,
}}, }},
{"s3:http://hostname:9999/bucket/prefix/directory", Config{ {S: "s3:http://hostname:9999/bucket/prefix/directory", Cfg: Config{
Endpoint: "hostname:9999", Endpoint: "hostname:9999",
Bucket: "bucket", Bucket: "bucket",
Prefix: "prefix/directory", Prefix: "prefix/directory",
UseHTTP: true, UseHTTP: true,
Connections: 5, Connections: 5,
}}, }},
{"s3:http://hostname:9999/bucket/prefix/directory/", Config{ {S: "s3:http://hostname:9999/bucket/prefix/directory/", Cfg: Config{
Endpoint: "hostname:9999", Endpoint: "hostname:9999",
Bucket: "bucket", Bucket: "bucket",
Prefix: "prefix/directory", Prefix: "prefix/directory",
@ -100,19 +99,7 @@ var configTests = []struct {
} }
func TestParseConfig(t *testing.T) { func TestParseConfig(t *testing.T) {
for i, test := range configTests { test.ParseConfigTester(t, ParseConfig, configTests)
cfg, err := ParseConfig(test.s)
if err != nil {
t.Errorf("test %d:%s failed: %v", i, test.s, err)
continue
}
if cfg != test.cfg {
t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v",
i, test.s, test.cfg, cfg)
continue
}
}
} }
func TestParseError(t *testing.T) { func TestParseError(t *testing.T) {

View file

@ -120,15 +120,15 @@ func createS3(t testing.TB, cfg MinioTestConfig, tr http.RoundTripper) (be resti
return be, err return be, err
} }
func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite { func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite[MinioTestConfig] {
tr, err := backend.Transport(backend.TransportOptions{}) tr, err := backend.Transport(backend.TransportOptions{})
if err != nil { if err != nil {
t.Fatalf("cannot create transport for tests: %v", err) t.Fatalf("cannot create transport for tests: %v", err)
} }
return &test.Suite{ return &test.Suite[MinioTestConfig]{
// NewConfig returns a config for a new temporary backend that will be used in tests. // NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) { NewConfig: func() (*MinioTestConfig, error) {
cfg := MinioTestConfig{} cfg := MinioTestConfig{}
cfg.tempdir = rtest.TempDir(t) cfg.tempdir = rtest.TempDir(t)
@ -142,13 +142,11 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
cfg.Config.UseHTTP = true cfg.Config.UseHTTP = true
cfg.Config.KeyID = key cfg.Config.KeyID = key
cfg.Config.Secret = options.NewSecretString(secret) cfg.Config.Secret = options.NewSecretString(secret)
return cfg, nil return &cfg, nil
}, },
// CreateFn is a function that creates a temporary repository for the tests. // CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) { Create: func(cfg MinioTestConfig) (restic.Backend, error) {
cfg := config.(MinioTestConfig)
be, err := createS3(t, cfg, tr) be, err := createS3(t, cfg, tr)
if err != nil { if err != nil {
return nil, err return nil, err
@ -167,14 +165,12 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
}, },
// OpenFn is a function that opens a previously created temporary repository. // OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) { Open: func(cfg MinioTestConfig) (restic.Backend, error) {
cfg := config.(MinioTestConfig)
return s3.Open(ctx, cfg.Config, tr) return s3.Open(ctx, cfg.Config, tr)
}, },
// CleanupFn removes data created during the tests. // CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error { Cleanup: func(cfg MinioTestConfig) error {
cfg := config.(MinioTestConfig)
if cfg.stopServer != nil { if cfg.stopServer != nil {
cfg.stopServer() cfg.stopServer()
} }
@ -217,24 +213,23 @@ func BenchmarkBackendMinio(t *testing.B) {
newMinioTestSuite(ctx, t).RunBenchmarks(t) newMinioTestSuite(ctx, t).RunBenchmarks(t)
} }
func newS3TestSuite(t testing.TB) *test.Suite { func newS3TestSuite(t testing.TB) *test.Suite[s3.Config] {
tr, err := backend.Transport(backend.TransportOptions{}) tr, err := backend.Transport(backend.TransportOptions{})
if err != nil { if err != nil {
t.Fatalf("cannot create transport for tests: %v", err) t.Fatalf("cannot create transport for tests: %v", err)
} }
return &test.Suite{ return &test.Suite[s3.Config]{
// do not use excessive data // do not use excessive data
MinimalData: true, MinimalData: true,
// NewConfig returns a config for a new temporary backend that will be used in tests. // NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) { NewConfig: func() (*s3.Config, error) {
s3cfg, err := s3.ParseConfig(os.Getenv("RESTIC_TEST_S3_REPOSITORY")) cfg, err := s3.ParseConfig(os.Getenv("RESTIC_TEST_S3_REPOSITORY"))
if err != nil { if err != nil {
return nil, err return nil, err
} }
cfg := s3cfg.(s3.Config)
cfg.KeyID = os.Getenv("RESTIC_TEST_S3_KEY") cfg.KeyID = os.Getenv("RESTIC_TEST_S3_KEY")
cfg.Secret = options.NewSecretString(os.Getenv("RESTIC_TEST_S3_SECRET")) cfg.Secret = options.NewSecretString(os.Getenv("RESTIC_TEST_S3_SECRET"))
cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
@ -242,9 +237,7 @@ func newS3TestSuite(t testing.TB) *test.Suite {
}, },
// CreateFn is a function that creates a temporary repository for the tests. // CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) { Create: func(cfg s3.Config) (restic.Backend, error) {
cfg := config.(s3.Config)
be, err := s3.Create(context.TODO(), cfg, tr) be, err := s3.Create(context.TODO(), cfg, tr)
if err != nil { if err != nil {
return nil, err return nil, err
@ -263,15 +256,12 @@ func newS3TestSuite(t testing.TB) *test.Suite {
}, },
// OpenFn is a function that opens a previously created temporary repository. // OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) { Open: func(cfg s3.Config) (restic.Backend, error) {
cfg := config.(s3.Config)
return s3.Open(context.TODO(), cfg, tr) return s3.Open(context.TODO(), cfg, tr)
}, },
// CleanupFn removes data created during the tests. // CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error { Cleanup: func(cfg s3.Config) error {
cfg := config.(s3.Config)
be, err := s3.Open(context.TODO(), cfg, tr) be, err := s3.Open(context.TODO(), cfg, tr)
if err != nil { if err != nil {
return err return err

View file

@ -35,7 +35,7 @@ func init() {
// and sftp:user@host:directory. The directory will be path Cleaned and can // and sftp:user@host:directory. The directory will be path Cleaned and can
// be an absolute path if it starts with a '/' (e.g. // be an absolute path if it starts with a '/' (e.g.
// sftp://user@host//absolute and sftp:user@host:/absolute). // sftp://user@host//absolute and sftp:user@host:/absolute).
func ParseConfig(s string) (interface{}, error) { func ParseConfig(s string) (*Config, error) {
var user, host, port, dir string var user, host, port, dir string
switch { switch {
case strings.HasPrefix(s, "sftp://"): case strings.HasPrefix(s, "sftp://"):
@ -89,5 +89,5 @@ func ParseConfig(s string) (interface{}, error) {
cfg.Port = port cfg.Port = port
cfg.Path = p cfg.Path = p
return cfg, nil return &cfg, nil
} }

View file

@ -2,94 +2,81 @@ package sftp
import ( import (
"testing" "testing"
"github.com/restic/restic/internal/backend/test"
) )
var configTests = []struct { var configTests = []test.ConfigTestData[Config]{
in string
cfg Config
}{
// first form, user specified sftp://user@host/dir // first form, user specified sftp://user@host/dir
{ {
"sftp://user@host/dir/subdir", S: "sftp://user@host/dir/subdir",
Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5}, Cfg: Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5},
}, },
{ {
"sftp://host/dir/subdir", S: "sftp://host/dir/subdir",
Config{Host: "host", Path: "dir/subdir", Connections: 5}, Cfg: Config{Host: "host", Path: "dir/subdir", Connections: 5},
}, },
{ {
"sftp://host//dir/subdir", S: "sftp://host//dir/subdir",
Config{Host: "host", Path: "/dir/subdir", Connections: 5}, Cfg: Config{Host: "host", Path: "/dir/subdir", Connections: 5},
}, },
{ {
"sftp://host:10022//dir/subdir", S: "sftp://host:10022//dir/subdir",
Config{Host: "host", Port: "10022", Path: "/dir/subdir", Connections: 5}, Cfg: Config{Host: "host", Port: "10022", Path: "/dir/subdir", Connections: 5},
}, },
{ {
"sftp://user@host:10022//dir/subdir", S: "sftp://user@host:10022//dir/subdir",
Config{User: "user", Host: "host", Port: "10022", Path: "/dir/subdir", Connections: 5}, Cfg: Config{User: "user", Host: "host", Port: "10022", Path: "/dir/subdir", Connections: 5},
}, },
{ {
"sftp://user@host/dir/subdir/../other", S: "sftp://user@host/dir/subdir/../other",
Config{User: "user", Host: "host", Path: "dir/other", Connections: 5}, Cfg: Config{User: "user", Host: "host", Path: "dir/other", Connections: 5},
}, },
{ {
"sftp://user@host/dir///subdir", S: "sftp://user@host/dir///subdir",
Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5}, Cfg: Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5},
}, },
// IPv6 address. // IPv6 address.
{ {
"sftp://user@[::1]/dir", S: "sftp://user@[::1]/dir",
Config{User: "user", Host: "::1", Path: "dir", Connections: 5}, Cfg: Config{User: "user", Host: "::1", Path: "dir", Connections: 5},
}, },
// IPv6 address with port. // IPv6 address with port.
{ {
"sftp://user@[::1]:22/dir", S: "sftp://user@[::1]:22/dir",
Config{User: "user", Host: "::1", Port: "22", Path: "dir", Connections: 5}, Cfg: Config{User: "user", Host: "::1", Port: "22", Path: "dir", Connections: 5},
}, },
// second form, user specified sftp:user@host:/dir // second form, user specified sftp:user@host:/dir
{ {
"sftp:user@host:/dir/subdir", S: "sftp:user@host:/dir/subdir",
Config{User: "user", Host: "host", Path: "/dir/subdir", Connections: 5}, Cfg: Config{User: "user", Host: "host", Path: "/dir/subdir", Connections: 5},
}, },
{ {
"sftp:user@domain@host:/dir/subdir", S: "sftp:user@domain@host:/dir/subdir",
Config{User: "user@domain", Host: "host", Path: "/dir/subdir", Connections: 5}, Cfg: Config{User: "user@domain", Host: "host", Path: "/dir/subdir", Connections: 5},
}, },
{ {
"sftp:host:../dir/subdir", S: "sftp:host:../dir/subdir",
Config{Host: "host", Path: "../dir/subdir", Connections: 5}, Cfg: Config{Host: "host", Path: "../dir/subdir", Connections: 5},
}, },
{ {
"sftp:user@host:dir/subdir:suffix", S: "sftp:user@host:dir/subdir:suffix",
Config{User: "user", Host: "host", Path: "dir/subdir:suffix", Connections: 5}, Cfg: Config{User: "user", Host: "host", Path: "dir/subdir:suffix", Connections: 5},
}, },
{ {
"sftp:user@host:dir/subdir/../other", S: "sftp:user@host:dir/subdir/../other",
Config{User: "user", Host: "host", Path: "dir/other", Connections: 5}, Cfg: Config{User: "user", Host: "host", Path: "dir/other", Connections: 5},
}, },
{ {
"sftp:user@host:dir///subdir", S: "sftp:user@host:dir///subdir",
Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5}, Cfg: Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5},
}, },
} }
func TestParseConfig(t *testing.T) { func TestParseConfig(t *testing.T) {
for i, test := range configTests { test.ParseConfigTester(t, ParseConfig, configTests)
cfg, err := ParseConfig(test.in)
if err != nil {
t.Errorf("test %d:%s failed: %v", i, test.in, err)
continue
}
if cfg != test.cfg {
t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v",
i, test.in, test.cfg, cfg)
continue
}
}
} }
var configTestsInvalid = []string{ var configTestsInvalid = []string{

View file

@ -29,10 +29,10 @@ func findSFTPServerBinary() string {
var sftpServer = findSFTPServerBinary() var sftpServer = findSFTPServerBinary()
func newTestSuite(t testing.TB) *test.Suite { func newTestSuite(t testing.TB) *test.Suite[sftp.Config] {
return &test.Suite{ return &test.Suite[sftp.Config]{
// NewConfig returns a config for a new temporary backend that will be used in tests. // NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) { NewConfig: func() (*sftp.Config, error) {
dir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-sftp-") dir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-sftp-")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -40,7 +40,7 @@ func newTestSuite(t testing.TB) *test.Suite {
t.Logf("create new backend at %v", dir) t.Logf("create new backend at %v", dir)
cfg := sftp.Config{ cfg := &sftp.Config{
Path: dir, Path: dir,
Command: fmt.Sprintf("%q -e", sftpServer), Command: fmt.Sprintf("%q -e", sftpServer),
Connections: 5, Connections: 5,
@ -49,20 +49,17 @@ func newTestSuite(t testing.TB) *test.Suite {
}, },
// CreateFn is a function that creates a temporary repository for the tests. // CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) { Create: func(cfg sftp.Config) (restic.Backend, error) {
cfg := config.(sftp.Config)
return sftp.Create(context.TODO(), cfg) return sftp.Create(context.TODO(), cfg)
}, },
// OpenFn is a function that opens a previously created temporary repository. // OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) { Open: func(cfg sftp.Config) (restic.Backend, error) {
cfg := config.(sftp.Config)
return sftp.Open(context.TODO(), cfg) return sftp.Open(context.TODO(), cfg)
}, },
// CleanupFn removes data created during the tests. // CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error { Cleanup: func(cfg sftp.Config) error {
cfg := config.(sftp.Config)
if !rtest.TestCleanupTempDirs { if !rtest.TestCleanupTempDirs {
t.Logf("leaving test backend dir at %v", cfg.Path) t.Logf("leaving test backend dir at %v", cfg.Path)
} }

View file

@ -2,6 +2,7 @@ package smb
import ( import (
"net/url" "net/url"
"os"
"path" "path"
"strconv" "strconv"
"strings" "strings"
@ -9,6 +10,7 @@ import (
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options" "github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
) )
// Config contains all configuration necessary to connect to an SMB server // Config contains all configuration necessary to connect to an SMB server
@ -54,7 +56,7 @@ func init() {
// ParseConfig parses the string s and extracts the s3 config. The // ParseConfig parses the string s and extracts the s3 config. The
// supported configuration format is smb://[user@]host[:port]/sharename/directory. // supported configuration format is smb://[user@]host[:port]/sharename/directory.
// User and port are optional. Default port is 445. // User and port are optional. Default port is 445.
func ParseConfig(s string) (interface{}, error) { func ParseConfig(s string) (*Config, error) {
var repo string var repo string
switch { switch {
case strings.HasPrefix(s, "smb://"): case strings.HasPrefix(s, "smb://"):
@ -106,7 +108,7 @@ func ParseConfig(s string) (interface{}, error) {
return createConfig(user, host, portNum, sharename, directory) return createConfig(user, host, portNum, sharename, directory)
} }
func createConfig(user string, host string, port int, sharename, directory string) (interface{}, error) { func createConfig(user string, host string, port int, sharename, directory string) (*Config, error) {
if host == "" { if host == "" {
return nil, errors.New("smb: invalid format, Host not found") return nil, errors.New("smb: invalid format, Host not found")
} }
@ -121,5 +123,26 @@ func createConfig(user string, host string, port int, sharename, directory strin
cfg.Port = port cfg.Port = port
cfg.ShareName = sharename cfg.ShareName = sharename
cfg.Path = directory cfg.Path = directory
return cfg, nil return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) error {
if cfg.User == "" {
cfg.User = os.Getenv("RESTIC_SMB_USER")
}
if cfg.Password.String() == "" {
cfg.Password = options.NewSecretString(os.Getenv("RESTIC_SMB_PASSWORD"))
}
if cfg.Domain == "" {
cfg.Domain = os.Getenv("RESTIC_SMB_DOMAIN")
}
if cfg.Domain == "" {
cfg.Domain = DefaultDomain
}
return nil
} }

View file

@ -3,76 +3,68 @@ package smb
import ( import (
"strings" "strings"
"testing" "testing"
"github.com/restic/restic/internal/backend/test"
) )
var configTests = []struct { var configTests = []test.ConfigTestData[Config]{
s string {S: "smb://user@host/sharename/directory",
cfg Config Cfg: Config{
}{ Host: "host",
{"smb://user@host/sharename/directory", Config{ Port: DefaultSmbPort,
Host: "host", User: "user",
Port: DefaultSmbPort, Domain: DefaultDomain,
User: "user", ShareName: "sharename",
Domain: DefaultDomain, Path: "directory",
ShareName: "sharename", Connections: DefaultConnections,
Path: "directory", IdleTimeout: DefaultIdleTimeout,
Connections: DefaultConnections, }},
IdleTimeout: DefaultIdleTimeout, {S: "smb://user@host:456/sharename/directory",
}}, Cfg: Config{
{"smb://user@host:456/sharename/directory", Config{ Host: "host",
Host: "host", Port: 456,
Port: 456, User: "user",
User: "user", Domain: DefaultDomain,
Domain: DefaultDomain, ShareName: "sharename",
ShareName: "sharename", Path: "directory",
Path: "directory", Connections: DefaultConnections,
Connections: DefaultConnections, IdleTimeout: DefaultIdleTimeout,
IdleTimeout: DefaultIdleTimeout, }},
}}, {S: "smb://host/sharename/directory",
{"smb://host/sharename/directory", Config{ Cfg: Config{
Host: "host", Host: "host",
Port: DefaultSmbPort, Port: DefaultSmbPort,
Domain: DefaultDomain, Domain: DefaultDomain,
ShareName: "sharename", ShareName: "sharename",
Path: "directory", Path: "directory",
Connections: DefaultConnections, Connections: DefaultConnections,
IdleTimeout: DefaultIdleTimeout, IdleTimeout: DefaultIdleTimeout,
}}, }},
{"smb://host:446/sharename/directory", Config{ {S: "smb://host:446/sharename/directory",
Host: "host", Cfg: Config{
Port: 446, Host: "host",
Domain: DefaultDomain, Port: 446,
ShareName: "sharename", Domain: DefaultDomain,
Path: "directory", ShareName: "sharename",
Connections: DefaultConnections, Path: "directory",
IdleTimeout: DefaultIdleTimeout, Connections: DefaultConnections,
}}, IdleTimeout: DefaultIdleTimeout,
{"smb:user@host:466/sharename/directory", Config{ }},
Host: "host", {S: "smb:user@host:466/sharename/directory",
Port: 466, Cfg: Config{
User: "user", Host: "host",
Domain: DefaultDomain, Port: 466,
ShareName: "sharename", User: "user",
Path: "directory", Domain: DefaultDomain,
Connections: DefaultConnections, ShareName: "sharename",
IdleTimeout: DefaultIdleTimeout, Path: "directory",
}}, Connections: DefaultConnections,
IdleTimeout: DefaultIdleTimeout,
}},
} }
func TestParseConfig(t *testing.T) { func TestParseConfig(t *testing.T) {
for i, test := range configTests { test.ParseConfigTester(t, ParseConfig, configTests)
cfg, err := ParseConfig(test.s)
if err != nil {
t.Errorf("test %d:%s failed: %v", i, test.s, err)
continue
}
if cfg != test.cfg {
t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v",
i, test.s, test.cfg, cfg)
continue
}
}
} }
func TestParseError(t *testing.T) { func TestParseError(t *testing.T) {

View file

@ -13,12 +13,12 @@ import (
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
func newTestSuite(t testing.TB) *test.Suite { func newTestSuite(t testing.TB) *test.Suite[smb.Config] {
return &test.Suite{ return &test.Suite[smb.Config]{
// NewConfig returns a config for a new temporary backend that will be used in tests. // NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) { NewConfig: func() (*smb.Config, error) {
cfg := smb.NewConfig() cfg := &smb.Config{}
cfg.Host = "127.0.0.1" cfg.Host = "127.0.0.1"
cfg.User = "smbuser" cfg.User = "smbuser"
cfg.ShareName = cfg.User cfg.ShareName = cfg.User
@ -38,20 +38,17 @@ func newTestSuite(t testing.TB) *test.Suite {
}, },
// CreateFn is a function that creates a temporary repository for the tests. // CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) { Create: func(cfg smb.Config) (restic.Backend, error) {
cfg := config.(smb.Config)
return smb.Create(context.TODO(), cfg) return smb.Create(context.TODO(), cfg)
}, },
// OpenFn is a function that opens a previously created temporary repository. // OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) { Open: func(cfg smb.Config) (restic.Backend, error) {
cfg := config.(smb.Config)
return smb.Open(context.TODO(), cfg) return smb.Open(context.TODO(), cfg)
}, },
// CleanupFn removes data created during the tests. // CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error { Cleanup: func(cfg smb.Config) error {
cfg := config.(smb.Config)
if !rtest.TestCleanupTempDirs { if !rtest.TestCleanupTempDirs {
t.Logf("leaving test backend dir at %v", cfg.Path) t.Logf("leaving test backend dir at %v", cfg.Path)
} }

View file

@ -6,6 +6,7 @@ import (
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options" "github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
) )
// Config contains basic configuration needed to specify swift location for a swift server // Config contains basic configuration needed to specify swift location for a swift server
@ -50,7 +51,7 @@ func NewConfig() Config {
} }
// ParseConfig parses the string s and extract swift's container name and prefix. // ParseConfig parses the string s and extract swift's container name and prefix.
func ParseConfig(s string) (interface{}, error) { func ParseConfig(s string) (*Config, error) {
if !strings.HasPrefix(s, "swift:") { if !strings.HasPrefix(s, "swift:") {
return nil, errors.New("invalid URL, expected: swift:container-name:/[prefix]") return nil, errors.New("invalid URL, expected: swift:container-name:/[prefix]")
} }
@ -70,48 +71,49 @@ func ParseConfig(s string) (interface{}, error) {
cfg.Container = container cfg.Container = container
cfg.Prefix = prefix cfg.Prefix = prefix
return cfg, nil return &cfg, nil
} }
var _ restic.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config. // ApplyEnvironment saves values from the environment to the config.
func ApplyEnvironment(prefix string, cfg interface{}) error { func (cfg *Config) ApplyEnvironment(prefix string) error {
c := cfg.(*Config)
for _, val := range []struct { for _, val := range []struct {
s *string s *string
env string env string
}{ }{
// v2/v3 specific // v2/v3 specific
{&c.UserName, prefix + "OS_USERNAME"}, {&cfg.UserName, prefix + "OS_USERNAME"},
{&c.APIKey, prefix + "OS_PASSWORD"}, {&cfg.APIKey, prefix + "OS_PASSWORD"},
{&c.Region, prefix + "OS_REGION_NAME"}, {&cfg.Region, prefix + "OS_REGION_NAME"},
{&c.AuthURL, prefix + "OS_AUTH_URL"}, {&cfg.AuthURL, prefix + "OS_AUTH_URL"},
// v3 specific // v3 specific
{&c.UserID, prefix + "OS_USER_ID"}, {&cfg.UserID, prefix + "OS_USER_ID"},
{&c.Domain, prefix + "OS_USER_DOMAIN_NAME"}, {&cfg.Domain, prefix + "OS_USER_DOMAIN_NAME"},
{&c.DomainID, prefix + "OS_USER_DOMAIN_ID"}, {&cfg.DomainID, prefix + "OS_USER_DOMAIN_ID"},
{&c.Tenant, prefix + "OS_PROJECT_NAME"}, {&cfg.Tenant, prefix + "OS_PROJECT_NAME"},
{&c.TenantDomain, prefix + "OS_PROJECT_DOMAIN_NAME"}, {&cfg.TenantDomain, prefix + "OS_PROJECT_DOMAIN_NAME"},
{&c.TenantDomainID, prefix + "OS_PROJECT_DOMAIN_ID"}, {&cfg.TenantDomainID, prefix + "OS_PROJECT_DOMAIN_ID"},
{&c.TrustID, prefix + "OS_TRUST_ID"}, {&cfg.TrustID, prefix + "OS_TRUST_ID"},
// v2 specific // v2 specific
{&c.TenantID, prefix + "OS_TENANT_ID"}, {&cfg.TenantID, prefix + "OS_TENANT_ID"},
{&c.Tenant, prefix + "OS_TENANT_NAME"}, {&cfg.Tenant, prefix + "OS_TENANT_NAME"},
// v1 specific // v1 specific
{&c.AuthURL, prefix + "ST_AUTH"}, {&cfg.AuthURL, prefix + "ST_AUTH"},
{&c.UserName, prefix + "ST_USER"}, {&cfg.UserName, prefix + "ST_USER"},
{&c.APIKey, prefix + "ST_KEY"}, {&cfg.APIKey, prefix + "ST_KEY"},
// Application Credential auth // Application Credential auth
{&c.ApplicationCredentialID, prefix + "OS_APPLICATION_CREDENTIAL_ID"}, {&cfg.ApplicationCredentialID, prefix + "OS_APPLICATION_CREDENTIAL_ID"},
{&c.ApplicationCredentialName, prefix + "OS_APPLICATION_CREDENTIAL_NAME"}, {&cfg.ApplicationCredentialName, prefix + "OS_APPLICATION_CREDENTIAL_NAME"},
// Manual authentication // Manual authentication
{&c.StorageURL, prefix + "OS_STORAGE_URL"}, {&cfg.StorageURL, prefix + "OS_STORAGE_URL"},
{&c.DefaultContainerPolicy, prefix + "SWIFT_DEFAULT_CONTAINER_POLICY"}, {&cfg.DefaultContainerPolicy, prefix + "SWIFT_DEFAULT_CONTAINER_POLICY"},
} { } {
if *val.s == "" { if *val.s == "" {
*val.s = os.Getenv(val.env) *val.s = os.Getenv(val.env)
@ -121,8 +123,8 @@ func ApplyEnvironment(prefix string, cfg interface{}) error {
s *options.SecretString s *options.SecretString
env string env string
}{ }{
{&c.ApplicationCredentialSecret, prefix + "OS_APPLICATION_CREDENTIAL_SECRET"}, {&cfg.ApplicationCredentialSecret, prefix + "OS_APPLICATION_CREDENTIAL_SECRET"},
{&c.AuthToken, prefix + "OS_AUTH_TOKEN"}, {&cfg.AuthToken, prefix + "OS_AUTH_TOKEN"},
} { } {
if val.s.String() == "" { if val.s.String() == "" {
*val.s = options.NewSecretString(os.Getenv(val.env)) *val.s = options.NewSecretString(os.Getenv(val.env))

View file

@ -1,29 +1,30 @@
package swift package swift
import "testing" import (
"testing"
var configTests = []struct { "github.com/restic/restic/internal/backend/test"
s string )
cfg Config
}{ var configTests = []test.ConfigTestData[Config]{
{ {
"swift:cnt1:/", S: "swift:cnt1:/",
Config{ Cfg: Config{
Container: "cnt1", Container: "cnt1",
Prefix: "", Prefix: "",
Connections: 5, Connections: 5,
}, },
}, },
{ {
"swift:cnt2:/prefix", S: "swift:cnt2:/prefix",
Config{Container: "cnt2", Cfg: Config{Container: "cnt2",
Prefix: "prefix", Prefix: "prefix",
Connections: 5, Connections: 5,
}, },
}, },
{ {
"swift:cnt3:/prefix/longer", S: "swift:cnt3:/prefix/longer",
Config{Container: "cnt3", Cfg: Config{Container: "cnt3",
Prefix: "prefix/longer", Prefix: "prefix/longer",
Connections: 5, Connections: 5,
}, },
@ -31,24 +32,7 @@ var configTests = []struct {
} }
func TestParseConfig(t *testing.T) { func TestParseConfig(t *testing.T) {
for _, test := range configTests { test.ParseConfigTester(t, ParseConfig, configTests)
t.Run("", func(t *testing.T) {
v, err := ParseConfig(test.s)
if err != nil {
t.Fatalf("parsing %q failed: %v", test.s, err)
}
cfg, ok := v.(Config)
if !ok {
t.Fatalf("wrong type returned, want Config, got %T", cfg)
}
if cfg != test.cfg {
t.Fatalf("wrong output for %q, want:\n %#v\ngot:\n %#v",
test.s, test.cfg, cfg)
}
})
}
} }
var configTestsInvalid = []string{ var configTestsInvalid = []string{

View file

@ -15,13 +15,13 @@ import (
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
func newSwiftTestSuite(t testing.TB) *test.Suite { func newSwiftTestSuite(t testing.TB) *test.Suite[swift.Config] {
tr, err := backend.Transport(backend.TransportOptions{}) tr, err := backend.Transport(backend.TransportOptions{})
if err != nil { if err != nil {
t.Fatalf("cannot create transport for tests: %v", err) t.Fatalf("cannot create transport for tests: %v", err)
} }
return &test.Suite{ return &test.Suite[swift.Config]{
// do not use excessive data // do not use excessive data
MinimalData: true, MinimalData: true,
@ -42,14 +42,13 @@ func newSwiftTestSuite(t testing.TB) *test.Suite {
}, },
// NewConfig returns a config for a new temporary backend that will be used in tests. // NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) { NewConfig: func() (*swift.Config, error) {
swiftcfg, err := swift.ParseConfig(os.Getenv("RESTIC_TEST_SWIFT")) cfg, err := swift.ParseConfig(os.Getenv("RESTIC_TEST_SWIFT"))
if err != nil { if err != nil {
return nil, err return nil, err
} }
cfg := swiftcfg.(swift.Config) if err = cfg.ApplyEnvironment("RESTIC_TEST_"); err != nil {
if err = swift.ApplyEnvironment("RESTIC_TEST_", &cfg); err != nil {
return nil, err return nil, err
} }
cfg.Prefix += fmt.Sprintf("/test-%d", time.Now().UnixNano()) cfg.Prefix += fmt.Sprintf("/test-%d", time.Now().UnixNano())
@ -58,9 +57,7 @@ func newSwiftTestSuite(t testing.TB) *test.Suite {
}, },
// CreateFn is a function that creates a temporary repository for the tests. // CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) { Create: func(cfg swift.Config) (restic.Backend, error) {
cfg := config.(swift.Config)
be, err := swift.Open(context.TODO(), cfg, tr) be, err := swift.Open(context.TODO(), cfg, tr)
if err != nil { if err != nil {
return nil, err return nil, err
@ -79,15 +76,12 @@ func newSwiftTestSuite(t testing.TB) *test.Suite {
}, },
// OpenFn is a function that opens a previously created temporary repository. // OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) { Open: func(cfg swift.Config) (restic.Backend, error) {
cfg := config.(swift.Config)
return swift.Open(context.TODO(), cfg, tr) return swift.Open(context.TODO(), cfg, tr)
}, },
// CleanupFn removes data created during the tests. // CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error { Cleanup: func(cfg swift.Config) error {
cfg := config.(swift.Config)
be, err := swift.Open(context.TODO(), cfg, tr) be, err := swift.Open(context.TODO(), cfg, tr)
if err != nil { if err != nil {
return err return err

View file

@ -29,7 +29,7 @@ func remove(t testing.TB, be restic.Backend, h restic.Handle) {
// BenchmarkLoadFile benchmarks the Load() method of a backend by // BenchmarkLoadFile benchmarks the Load() method of a backend by
// loading a complete file. // loading a complete file.
func (s *Suite) BenchmarkLoadFile(t *testing.B) { func (s *Suite[C]) BenchmarkLoadFile(t *testing.B) {
be := s.open(t) be := s.open(t)
defer s.close(t, be) defer s.close(t, be)
@ -64,7 +64,7 @@ func (s *Suite) BenchmarkLoadFile(t *testing.B) {
// BenchmarkLoadPartialFile benchmarks the Load() method of a backend by // BenchmarkLoadPartialFile benchmarks the Load() method of a backend by
// loading the remainder of a file starting at a given offset. // loading the remainder of a file starting at a given offset.
func (s *Suite) BenchmarkLoadPartialFile(t *testing.B) { func (s *Suite[C]) BenchmarkLoadPartialFile(t *testing.B) {
be := s.open(t) be := s.open(t)
defer s.close(t, be) defer s.close(t, be)
@ -101,7 +101,7 @@ func (s *Suite) BenchmarkLoadPartialFile(t *testing.B) {
// BenchmarkLoadPartialFileOffset benchmarks the Load() method of a // BenchmarkLoadPartialFileOffset benchmarks the Load() method of a
// backend by loading a number of bytes of a file starting at a given offset. // backend by loading a number of bytes of a file starting at a given offset.
func (s *Suite) BenchmarkLoadPartialFileOffset(t *testing.B) { func (s *Suite[C]) BenchmarkLoadPartialFileOffset(t *testing.B) {
be := s.open(t) be := s.open(t)
defer s.close(t, be) defer s.close(t, be)
@ -139,7 +139,7 @@ func (s *Suite) BenchmarkLoadPartialFileOffset(t *testing.B) {
} }
// BenchmarkSave benchmarks the Save() method of a backend. // BenchmarkSave benchmarks the Save() method of a backend.
func (s *Suite) BenchmarkSave(t *testing.B) { func (s *Suite[C]) BenchmarkSave(t *testing.B) {
be := s.open(t) be := s.open(t)
defer s.close(t, be) defer s.close(t, be)

View file

@ -0,0 +1,28 @@
package test
import (
"fmt"
"reflect"
"testing"
)
type ConfigTestData[C comparable] struct {
S string
Cfg C
}
func ParseConfigTester[C comparable](t *testing.T, parser func(s string) (*C, error), tests []ConfigTestData[C]) {
for i, test := range tests {
t.Run(fmt.Sprint(i), func(t *testing.T) {
cfg, err := parser(test.S)
if err != nil {
t.Fatalf("%s failed: %v", test.S, err)
}
if !reflect.DeepEqual(*cfg, test.Cfg) {
t.Fatalf("input: %s\n wrong config, want:\n %#v\ngot:\n %#v",
test.S, test.Cfg, *cfg)
}
})
}
}

View file

@ -11,21 +11,21 @@ import (
) )
// Suite implements a test suite for restic backends. // Suite implements a test suite for restic backends.
type Suite struct { type Suite[C any] struct {
// Config should be used to configure the backend. // Config should be used to configure the backend.
Config interface{} Config *C
// NewConfig returns a config for a new temporary backend that will be used in tests. // NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig func() (interface{}, error) NewConfig func() (*C, error)
// CreateFn is a function that creates a temporary repository for the tests. // CreateFn is a function that creates a temporary repository for the tests.
Create func(cfg interface{}) (restic.Backend, error) Create func(cfg C) (restic.Backend, error)
// OpenFn is a function that opens a previously created temporary repository. // OpenFn is a function that opens a previously created temporary repository.
Open func(cfg interface{}) (restic.Backend, error) Open func(cfg C) (restic.Backend, error)
// CleanupFn removes data created during the tests. // CleanupFn removes data created during the tests.
Cleanup func(cfg interface{}) error Cleanup func(cfg C) error
// MinimalData instructs the tests to not use excessive data. // MinimalData instructs the tests to not use excessive data.
MinimalData bool MinimalData bool
@ -40,7 +40,7 @@ type Suite struct {
} }
// RunTests executes all defined tests as subtests of t. // RunTests executes all defined tests as subtests of t.
func (s *Suite) RunTests(t *testing.T) { func (s *Suite[C]) RunTests(t *testing.T) {
var err error var err error
s.Config, err = s.NewConfig() s.Config, err = s.NewConfig()
if err != nil { if err != nil {
@ -61,7 +61,7 @@ func (s *Suite) RunTests(t *testing.T) {
} }
if s.Cleanup != nil { if s.Cleanup != nil {
if err = s.Cleanup(s.Config); err != nil { if err = s.Cleanup(*s.Config); err != nil {
t.Fatal(err) t.Fatal(err)
} }
} }
@ -72,7 +72,7 @@ type testFunction struct {
Fn func(*testing.T) Fn func(*testing.T)
} }
func (s *Suite) testFuncs(t testing.TB) (funcs []testFunction) { func (s *Suite[C]) testFuncs(t testing.TB) (funcs []testFunction) {
tpe := reflect.TypeOf(s) tpe := reflect.TypeOf(s)
v := reflect.ValueOf(s) v := reflect.ValueOf(s)
@ -107,7 +107,7 @@ type benchmarkFunction struct {
Fn func(*testing.B) Fn func(*testing.B)
} }
func (s *Suite) benchmarkFuncs(t testing.TB) (funcs []benchmarkFunction) { func (s *Suite[C]) benchmarkFuncs(t testing.TB) (funcs []benchmarkFunction) {
tpe := reflect.TypeOf(s) tpe := reflect.TypeOf(s)
v := reflect.ValueOf(s) v := reflect.ValueOf(s)
@ -138,7 +138,7 @@ func (s *Suite) benchmarkFuncs(t testing.TB) (funcs []benchmarkFunction) {
} }
// RunBenchmarks executes all defined benchmarks as subtests of b. // RunBenchmarks executes all defined benchmarks as subtests of b.
func (s *Suite) RunBenchmarks(b *testing.B) { func (s *Suite[C]) RunBenchmarks(b *testing.B) {
var err error var err error
s.Config, err = s.NewConfig() s.Config, err = s.NewConfig()
if err != nil { if err != nil {
@ -158,28 +158,28 @@ func (s *Suite) RunBenchmarks(b *testing.B) {
return return
} }
if err = s.Cleanup(s.Config); err != nil { if err = s.Cleanup(*s.Config); err != nil {
b.Fatal(err) b.Fatal(err)
} }
} }
func (s *Suite) create(t testing.TB) restic.Backend { func (s *Suite[C]) create(t testing.TB) restic.Backend {
be, err := s.Create(s.Config) be, err := s.Create(*s.Config)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
return be return be
} }
func (s *Suite) open(t testing.TB) restic.Backend { func (s *Suite[C]) open(t testing.TB) restic.Backend {
be, err := s.Open(s.Config) be, err := s.Open(*s.Config)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
return be return be
} }
func (s *Suite) close(t testing.TB, be restic.Backend) { func (s *Suite[C]) close(t testing.TB, be restic.Backend) {
err := be.Close() err := be.Close()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)

View file

@ -38,7 +38,7 @@ func beTest(ctx context.Context, be restic.Backend, h restic.Handle) (bool, erro
// TestCreateWithConfig tests that creating a backend in a location which already // TestCreateWithConfig tests that creating a backend in a location which already
// has a config file fails. // has a config file fails.
func (s *Suite) TestCreateWithConfig(t *testing.T) { func (s *Suite[C]) TestCreateWithConfig(t *testing.T) {
b := s.open(t) b := s.open(t)
defer s.close(t, b) defer s.close(t, b)
@ -57,7 +57,7 @@ func (s *Suite) TestCreateWithConfig(t *testing.T) {
store(t, b, restic.ConfigFile, []byte("test config")) store(t, b, restic.ConfigFile, []byte("test config"))
// now create the backend again, this must fail // now create the backend again, this must fail
_, err = s.Create(s.Config) _, err = s.Create(*s.Config)
if err == nil { if err == nil {
t.Fatalf("expected error not found for creating a backend with an existing config file") t.Fatalf("expected error not found for creating a backend with an existing config file")
} }
@ -70,7 +70,7 @@ func (s *Suite) TestCreateWithConfig(t *testing.T) {
} }
// TestLocation tests that a location string is returned. // TestLocation tests that a location string is returned.
func (s *Suite) TestLocation(t *testing.T) { func (s *Suite[C]) TestLocation(t *testing.T) {
b := s.open(t) b := s.open(t)
defer s.close(t, b) defer s.close(t, b)
@ -81,7 +81,7 @@ func (s *Suite) TestLocation(t *testing.T) {
} }
// TestConfig saves and loads a config from the backend. // TestConfig saves and loads a config from the backend.
func (s *Suite) TestConfig(t *testing.T) { func (s *Suite[C]) TestConfig(t *testing.T) {
b := s.open(t) b := s.open(t)
defer s.close(t, b) defer s.close(t, b)
@ -118,7 +118,7 @@ func (s *Suite) TestConfig(t *testing.T) {
} }
// TestLoad tests the backend's Load function. // TestLoad tests the backend's Load function.
func (s *Suite) TestLoad(t *testing.T) { func (s *Suite[C]) TestLoad(t *testing.T) {
seedRand(t) seedRand(t)
b := s.open(t) b := s.open(t)
@ -222,8 +222,12 @@ func (s *Suite) TestLoad(t *testing.T) {
test.OK(t, b.Remove(context.TODO(), handle)) test.OK(t, b.Remove(context.TODO(), handle))
} }
type setter interface {
SetListMaxItems(int)
}
// TestList makes sure that the backend implements List() pagination correctly. // TestList makes sure that the backend implements List() pagination correctly.
func (s *Suite) TestList(t *testing.T) { func (s *Suite[C]) TestList(t *testing.T) {
seedRand(t) seedRand(t)
numTestFiles := rand.Intn(20) + 20 numTestFiles := rand.Intn(20) + 20
@ -269,10 +273,6 @@ func (s *Suite) TestList(t *testing.T) {
t.Run(fmt.Sprintf("max-%v", test.maxItems), func(t *testing.T) { t.Run(fmt.Sprintf("max-%v", test.maxItems), func(t *testing.T) {
list2 := make(map[restic.ID]int64) list2 := make(map[restic.ID]int64)
type setter interface {
SetListMaxItems(int)
}
if s, ok := b.(setter); ok { if s, ok := b.(setter); ok {
t.Logf("setting max list items to %d", test.maxItems) t.Logf("setting max list items to %d", test.maxItems)
s.SetListMaxItems(test.maxItems) s.SetListMaxItems(test.maxItems)
@ -326,7 +326,7 @@ func (s *Suite) TestList(t *testing.T) {
} }
// TestListCancel tests that the context is respected and the error is returned by List. // TestListCancel tests that the context is respected and the error is returned by List.
func (s *Suite) TestListCancel(t *testing.T) { func (s *Suite[C]) TestListCancel(t *testing.T) {
seedRand(t) seedRand(t)
numTestFiles := 5 numTestFiles := 5
@ -466,7 +466,7 @@ func (ec errorCloser) Rewind() error {
} }
// TestSave tests saving data in the backend. // TestSave tests saving data in the backend.
func (s *Suite) TestSave(t *testing.T) { func (s *Suite[C]) TestSave(t *testing.T) {
seedRand(t) seedRand(t)
b := s.open(t) b := s.open(t)
@ -582,7 +582,7 @@ func (r *incompleteByteReader) Length() int64 {
} }
// TestSaveError tests saving data in the backend. // TestSaveError tests saving data in the backend.
func (s *Suite) TestSaveError(t *testing.T) { func (s *Suite[C]) TestSaveError(t *testing.T) {
seedRand(t) seedRand(t)
b := s.open(t) b := s.open(t)
@ -621,7 +621,7 @@ func (b *wrongByteReader) Hash() []byte {
} }
// TestSaveWrongHash tests that uploads with a wrong hash fail // TestSaveWrongHash tests that uploads with a wrong hash fail
func (s *Suite) TestSaveWrongHash(t *testing.T) { func (s *Suite[C]) TestSaveWrongHash(t *testing.T) {
seedRand(t) seedRand(t)
b := s.open(t) b := s.open(t)
@ -679,7 +679,7 @@ func testLoad(b restic.Backend, h restic.Handle) error {
}) })
} }
func (s *Suite) delayedRemove(t testing.TB, be restic.Backend, handles ...restic.Handle) error { func (s *Suite[C]) delayedRemove(t testing.TB, be restic.Backend, handles ...restic.Handle) error {
// Some backend (swift, I'm looking at you) may implement delayed // Some backend (swift, I'm looking at you) may implement delayed
// removal of data. Let's wait a bit if this happens. // removal of data. Let's wait a bit if this happens.
@ -746,7 +746,7 @@ func delayedList(t testing.TB, b restic.Backend, tpe restic.FileType, max int, m
} }
// TestBackend tests all functions of the backend. // TestBackend tests all functions of the backend.
func (s *Suite) TestBackend(t *testing.T) { func (s *Suite[C]) TestBackend(t *testing.T) {
b := s.open(t) b := s.open(t)
defer s.close(t, b) defer s.close(t, b)
@ -867,7 +867,7 @@ func (s *Suite) TestBackend(t *testing.T) {
} }
// TestZZZDelete tests the Delete function. The name ensures that this test is executed last. // TestZZZDelete tests the Delete function. The name ensures that this test is executed last.
func (s *Suite) TestZZZDelete(t *testing.T) { func (s *Suite[C]) TestZZZDelete(t *testing.T) {
if !test.TestCleanupTempDirs { if !test.TestCleanupTempDirs {
t.Skipf("not removing backend, TestCleanupTempDirs is false") t.Skipf("not removing backend, TestCleanupTempDirs is false")
} }

View file

@ -1,4 +1,4 @@
package restorer package fs
import ( import (
"os" "os"
@ -6,7 +6,7 @@ import (
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
) )
func preallocateFile(wr *os.File, size int64) error { func PreallocateFile(wr *os.File, size int64) error {
// try contiguous first // try contiguous first
fst := unix.Fstore_t{ fst := unix.Fstore_t{
Flags: unix.F_ALLOCATECONTIG | unix.F_ALLOCATEALL, Flags: unix.F_ALLOCATECONTIG | unix.F_ALLOCATEALL,

View file

@ -1,4 +1,4 @@
package restorer package fs
import ( import (
"os" "os"
@ -6,7 +6,7 @@ import (
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
) )
func preallocateFile(wr *os.File, size int64) error { func PreallocateFile(wr *os.File, size int64) error {
if size <= 0 { if size <= 0 {
return nil return nil
} }

View file

@ -1,11 +1,11 @@
//go:build !linux && !darwin //go:build !linux && !darwin
// +build !linux,!darwin // +build !linux,!darwin
package restorer package fs
import "os" import "os"
func preallocateFile(wr *os.File, size int64) error { func PreallocateFile(wr *os.File, size int64) error {
// Maybe truncate can help? // Maybe truncate can help?
// Windows: This calls SetEndOfFile which preallocates space on disk // Windows: This calls SetEndOfFile which preallocates space on disk
return wr.Truncate(size) return wr.Truncate(size)

View file

@ -1,4 +1,4 @@
package restorer package fs
import ( import (
"os" "os"
@ -7,7 +7,6 @@ import (
"syscall" "syscall"
"testing" "testing"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/test" "github.com/restic/restic/internal/test"
) )
@ -23,7 +22,7 @@ func TestPreallocate(t *testing.T) {
test.OK(t, wr.Close()) test.OK(t, wr.Close())
}() }()
err = preallocateFile(wr, i) err = PreallocateFile(wr, i)
if err == syscall.ENOTSUP { if err == syscall.ENOTSUP {
t.SkipNow() t.SkipNow()
} }
@ -32,7 +31,7 @@ func TestPreallocate(t *testing.T) {
fi, err := wr.Stat() fi, err := wr.Stat()
test.OK(t, err) test.OK(t, err)
efi := fs.ExtendedStat(fi) efi := ExtendedStat(fi)
test.Assert(t, efi.Size == i || efi.Blocks > 0, "Preallocated size of %v, got size %v block %v", i, efi.Size, efi.Blocks) test.Assert(t, efi.Size == i || efi.Blocks > 0, "Preallocated size of %v, got size %v block %v", i, efi.Size, efi.Blocks)
}) })
} }

View file

@ -80,3 +80,8 @@ type FileInfo struct {
Size int64 Size int64
Name string Name string
} }
// ApplyEnvironmenter fills in a backend configuration from the environment
type ApplyEnvironmenter interface {
ApplyEnvironment(prefix string) error
}

View file

@ -6,6 +6,7 @@ import (
"github.com/cespare/xxhash/v2" "github.com/cespare/xxhash/v2"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/fs"
) )
// writes blobs to target files. // writes blobs to target files.
@ -72,7 +73,7 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create
return nil, err return nil, err
} }
} else { } else {
err := preallocateFile(wr.File, createSize) err := fs.PreallocateFile(wr.File, createSize)
if err != nil { if err != nil {
// Just log the preallocate error but don't let it cause the restore process to fail. // Just log the preallocate error but don't let it cause the restore process to fail.
// Preallocate might return an error if the filesystem (implementation) does not // Preallocate might return an error if the filesystem (implementation) does not

View file

@ -43,7 +43,8 @@ type Progress struct {
progress.Updater progress.Updater
mu sync.Mutex mu sync.Mutex
start time.Time start time.Time
estimator rateEstimator
scanStarted, scanFinished bool scanStarted, scanFinished bool
@ -60,6 +61,7 @@ func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress {
start: time.Now(), start: time.Now(),
currentFiles: make(map[string]struct{}), currentFiles: make(map[string]struct{}),
printer: printer, printer: printer,
estimator: *newRateEstimator(time.Now()),
} }
p.Updater = *progress.NewUpdater(interval, func(runtime time.Duration, final bool) { p.Updater = *progress.NewUpdater(interval, func(runtime time.Duration, final bool) {
if final { if final {
@ -73,9 +75,14 @@ func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress {
var secondsRemaining uint64 var secondsRemaining uint64
if p.scanFinished { if p.scanFinished {
secs := float64(runtime / time.Second) rate := p.estimator.rate(time.Now())
todo := float64(p.total.Bytes - p.processed.Bytes) tooSlowCutoff := 1024.
secondsRemaining = uint64(secs / float64(p.processed.Bytes) * todo) if rate <= tooSlowCutoff {
secondsRemaining = 0
} else {
todo := float64(p.total.Bytes - p.processed.Bytes)
secondsRemaining = uint64(todo / rate)
}
} }
p.printer.Update(p.total, p.processed, p.errors, p.currentFiles, p.start, secondsRemaining) p.printer.Update(p.total, p.processed, p.errors, p.currentFiles, p.start, secondsRemaining)
@ -105,6 +112,7 @@ func (p *Progress) addProcessed(c Counter) {
p.processed.Files += c.Files p.processed.Files += c.Files
p.processed.Dirs += c.Dirs p.processed.Dirs += c.Dirs
p.processed.Bytes += c.Bytes p.processed.Bytes += c.Bytes
p.estimator.recordBytes(time.Now(), c.Bytes)
p.scanStarted = true p.scanStarted = true
} }

View file

@ -0,0 +1,98 @@
package backup
import (
"container/list"
"time"
)
// rateBucket represents a one second window of recorded progress.
type rateBucket struct {
totalBytes uint64
end time.Time // the end of the time window, exclusive
}
// rateEstimator represents an estimate of the time to complete an operation.
type rateEstimator struct {
buckets *list.List
start time.Time
totalBytes uint64
}
// newRateEstimator returns an estimator initialized to a presumed start time.
func newRateEstimator(start time.Time) *rateEstimator {
return &rateEstimator{buckets: list.New(), start: start}
}
// See trim(), below.
const (
bucketWidth = time.Second
minRateEstimatorBytes = 100 * 1000 * 1000
minRateEstimatorBuckets = 20
minRateEstimatorMinutes = 2
)
// trim removes the oldest history from the estimator assuming a given
// current time.
func (r *rateEstimator) trim(now time.Time) {
// The estimator retains byte transfer counts over a two minute window.
// However, to avoid removing too much history when transfer rates are
// low, the estimator also retains a minimum number of processed bytes
// across a minimum number of buckets. An operation that is processing a
// significant number of bytes per second will typically retain only a
// two minute window's worth of information. One that is making slow
// progress, such as one being over a rate limited connection, typically
// observes bursts of updates as infrequently as every ten or twenty
// seconds, in which case the other limiters will kick in. This heuristic
// avoids wildly fluctuating estimates over rate limited connections.
start := now.Add(-minRateEstimatorMinutes * time.Minute)
for e := r.buckets.Front(); e != nil; e = r.buckets.Front() {
if r.buckets.Len() <= minRateEstimatorBuckets {
break
}
b := e.Value.(*rateBucket)
if b.end.After(start) {
break
}
total := r.totalBytes - b.totalBytes
if total < minRateEstimatorBytes {
break
}
r.start = b.end
r.totalBytes = total
r.buckets.Remove(e)
}
}
// recordBytes records the transfer of a number of bytes at a given
// time. Times passed in successive calls should advance monotonically (as
// is the case with time.Now().
func (r *rateEstimator) recordBytes(now time.Time, bytes uint64) {
if bytes == 0 {
return
}
var tail *rateBucket
if r.buckets.Len() > 0 {
tail = r.buckets.Back().Value.(*rateBucket)
}
if tail == nil || !tail.end.After(now) {
// The new bucket holds measurements in the time range [now .. now+1sec).
tail = &rateBucket{end: now.Add(bucketWidth)}
r.buckets.PushBack(tail)
}
tail.totalBytes += bytes
r.totalBytes += bytes
r.trim(now)
}
// rate returns an estimated bytes per second rate at a given time, or zero
// if there is not enough data to compute a rate.
func (r *rateEstimator) rate(now time.Time) float64 {
r.trim(now)
if !r.start.Before(now) {
return 0
}
elapsed := float64(now.Sub(r.start)) / float64(time.Second)
rate := float64(r.totalBytes) / elapsed
return rate
}

View file

@ -0,0 +1,213 @@
package backup
import (
"fmt"
"math"
"testing"
"time"
rtest "github.com/restic/restic/internal/test"
)
const float64EqualityThreshold = 1e-6
func almostEqual(a, b float64) bool {
if math.IsNaN(a) || math.IsNaN(b) {
panic("almostEqual passed a NaN")
}
return math.Abs(a-b) <= float64EqualityThreshold
}
func TestEstimatorDefault(t *testing.T) {
var start time.Time
e := newRateEstimator(start)
r := e.rate(start)
rtest.Assert(t, r == 0, "e.Rate == %v, want zero", r)
r = e.rate(start.Add(time.Hour))
rtest.Assert(t, r == 0, "e.Rate == %v, want zero", r)
}
func TestEstimatorSimple(t *testing.T) {
var start time.Time
type testcase struct {
bytes uint64
when time.Duration
rate float64
}
cases := []testcase{
{0, 0, 0},
{1, time.Second, 1},
{60, time.Second, 60},
{60, time.Minute, 1},
}
for _, c := range cases {
name := fmt.Sprintf("%+v", c)
t.Run(name, func(t *testing.T) {
e := newRateEstimator(start)
e.recordBytes(start.Add(time.Second), c.bytes)
rate := e.rate(start.Add(c.when))
rtest.Assert(t, almostEqual(rate, c.rate), "e.Rate == %v, want %v", rate, c.rate)
})
}
}
func TestBucketWidth(t *testing.T) {
var when time.Time
// Recording byte transfers within a bucket width's time window uses one
// bucket.
e := newRateEstimator(when)
e.recordBytes(when, 1)
e.recordBytes(when.Add(bucketWidth-time.Nanosecond), 1)
rtest.Assert(t, e.buckets.Len() == 1, "e.buckets.Len() is %d, want 1", e.buckets.Len())
b := e.buckets.Back().Value.(*rateBucket)
rtest.Assert(t, b.totalBytes == 2, "b.totalBytes is %d, want 2", b.totalBytes)
rtest.Assert(t, b.end == when.Add(bucketWidth), "b.end is %v, want %v", b.end, when.Add(bucketWidth))
// Recording a byte outside the bucket width causes another bucket.
e.recordBytes(when.Add(bucketWidth), 1)
rtest.Assert(t, e.buckets.Len() == 2, "e.buckets.Len() is %d, want 2", e.buckets.Len())
b = e.buckets.Back().Value.(*rateBucket)
rtest.Assert(t, b.totalBytes == 1, "b.totalBytes is %d, want 1", b.totalBytes)
rtest.Assert(t, b.end == when.Add(2*bucketWidth), "b.end is %v, want %v", b.end, when.Add(bucketWidth))
// Recording a byte after a longer delay creates a sparse bucket list.
e.recordBytes(when.Add(time.Hour+time.Millisecond), 7)
rtest.Assert(t, e.buckets.Len() == 3, "e.buckets.Len() is %d, want 3", e.buckets.Len())
b = e.buckets.Back().Value.(*rateBucket)
rtest.Assert(t, b.totalBytes == 7, "b.totalBytes is %d, want 7", b.totalBytes)
rtest.Equals(t, when.Add(time.Hour+time.Millisecond+time.Second), b.end)
}
type chunk struct {
repetitions uint64 // repetition count
bytes uint64 // byte count (every second)
}
func applyChunks(chunks []chunk, t time.Time, e *rateEstimator) time.Time {
for _, c := range chunks {
for i := uint64(0); i < c.repetitions; i++ {
e.recordBytes(t, c.bytes)
t = t.Add(time.Second)
}
}
return t
}
func TestEstimatorResponsiveness(t *testing.T) {
type testcase struct {
description string
chunks []chunk
rate float64
}
cases := []testcase{
{
"1000 bytes/sec over one second",
[]chunk{
{1, 1000},
},
1000,
},
{
"1000 bytes/sec over one minute",
[]chunk{
{60, 1000},
},
1000,
},
{
"1000 bytes/sec for 10 seconds, then 2000 bytes/sec for 10 seconds",
[]chunk{
{10, 1000},
{10, 2000},
},
1500,
},
{
"1000 bytes/sec for one minute, then 2000 bytes/sec for one minute",
[]chunk{
{60, 1000},
{60, 2000},
},
1500,
},
{
"rate doubles after 30 seconds",
[]chunk{
{30, minRateEstimatorBytes},
{90, 2 * minRateEstimatorBytes},
},
minRateEstimatorBytes * 1.75,
},
{
"rate doubles after 31 seconds",
[]chunk{
{31, minRateEstimatorBytes},
{90, 2 * minRateEstimatorBytes},
},
// The expected rate is the same as the prior test case because the
// first second has rolled off the estimator.
minRateEstimatorBytes * 1.75,
},
{
"rate doubles after 90 seconds",
[]chunk{
{90, minRateEstimatorBytes},
{90, 2 * minRateEstimatorBytes},
},
// The expected rate is the same as the prior test case because the
// first 60 seconds have rolled off the estimator.
minRateEstimatorBytes * 1.75,
},
{
"rate doubles for two full minutes",
[]chunk{
{60, minRateEstimatorBytes},
{120, 2 * minRateEstimatorBytes},
},
2 * minRateEstimatorBytes,
},
{
"rate falls to zero",
[]chunk{
{30, minRateEstimatorBytes},
{30, 0},
},
minRateEstimatorBytes / 2,
},
{
"rate falls to zero for extended time",
[]chunk{
{60, 1000},
{300, 0},
},
1000 * 60 / (60 + 300.0),
},
{
"rate falls to zero for extended time (from high rate)",
[]chunk{
{2 * minRateEstimatorBuckets, minRateEstimatorBytes},
{300, 0},
},
// Expect that only minRateEstimatorBuckets buckets are used in the
// rate estimate.
minRateEstimatorBytes * minRateEstimatorBuckets /
(minRateEstimatorBuckets + 300.0),
},
}
for _, c := range cases {
t.Run(c.description, func(t *testing.T) {
var w time.Time
e := newRateEstimator(w)
w = applyChunks(c.chunks, w, e)
r := e.rate(w)
rtest.Assert(t, almostEqual(r, c.rate), "e.Rate == %f, want %f", r, c.rate)
})
}
}