diff --git a/changelog/unreleased/issue-3397 b/changelog/unreleased/issue-3397 new file mode 100644 index 000000000..391eeb004 --- /dev/null +++ b/changelog/unreleased/issue-3397 @@ -0,0 +1,11 @@ +Enhancement: Improve the ETA displayed during backup + +Restic's `backup` command displayed an ETA that did not adapt when the rate +of progress made during the backup changed during the course of the +backup. Restic now uses recent progress when computing the ETA. It is +important to realize that the estimate may still be wrong, because restic +cannot predict the future, but the hope is that the ETA will be more +accurate in most cases. + +https://github.com/restic/restic/issues/3397 +https://github.com/restic/restic/pull/3563 diff --git a/changelog/unreleased/pull-3261 b/changelog/unreleased/pull-3261 new file mode 100644 index 000000000..f7073ed7b --- /dev/null +++ b/changelog/unreleased/pull-3261 @@ -0,0 +1,8 @@ +Enhancement: Reduce file fragmentation for local backend + +Before this change, local backend files could become fragmented. +Now restic will try to preallocate space for pack files to avoid +their fragmentation. + +https://github.com/restic/restic/issues/2679 +https://github.com/restic/restic/pull/3261 diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 792cf23ac..e898ad9ad 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -536,170 +536,21 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi } func parseConfig(loc location.Location, opts options.Options) (interface{}, error) { - // only apply options for a particular backend here - opts = opts.Extract(loc.Scheme) - - switch loc.Scheme { - case "local": - cfg := loc.Config.(local.Config) - if err := opts.Apply(loc.Scheme, &cfg); err != nil { + cfg := loc.Config + if cfg, ok := cfg.(restic.ApplyEnvironmenter); ok { + if err := cfg.ApplyEnvironment(""); err != nil { return nil, err } - - debug.Log("opening local repository at %#v", cfg) - return cfg, nil - - case "sftp": - cfg := loc.Config.(sftp.Config) - if err := opts.Apply(loc.Scheme, &cfg); err != nil { - return nil, err - } - - debug.Log("opening sftp repository at %#v", cfg) - return cfg, nil - - case "s3": - cfg := loc.Config.(s3.Config) - if cfg.KeyID == "" { - cfg.KeyID = os.Getenv("AWS_ACCESS_KEY_ID") - } - - if cfg.Secret.String() == "" { - cfg.Secret = options.NewSecretString(os.Getenv("AWS_SECRET_ACCESS_KEY")) - } - - if cfg.KeyID == "" && cfg.Secret.String() != "" { - return nil, errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty") - } else if cfg.KeyID != "" && cfg.Secret.String() == "" { - return nil, errors.Fatalf("unable to open S3 backend: Secret ($AWS_SECRET_ACCESS_KEY) is empty") - } - - if cfg.Region == "" { - cfg.Region = os.Getenv("AWS_DEFAULT_REGION") - } - - if err := opts.Apply(loc.Scheme, &cfg); err != nil { - return nil, err - } - - debug.Log("opening s3 repository at %#v", cfg) - return cfg, nil - - case "gs": - cfg := loc.Config.(gs.Config) - if cfg.ProjectID == "" { - cfg.ProjectID = os.Getenv("GOOGLE_PROJECT_ID") - } - - if err := opts.Apply(loc.Scheme, &cfg); err != nil { - return nil, err - } - - debug.Log("opening gs repository at %#v", cfg) - return cfg, nil - - case "azure": - cfg := loc.Config.(azure.Config) - if cfg.AccountName == "" { - cfg.AccountName = os.Getenv("AZURE_ACCOUNT_NAME") - } - - if cfg.AccountKey.String() == "" { - cfg.AccountKey = options.NewSecretString(os.Getenv("AZURE_ACCOUNT_KEY")) - } - - if cfg.AccountSAS.String() == "" { - cfg.AccountSAS = options.NewSecretString(os.Getenv("AZURE_ACCOUNT_SAS")) - } - - if err := opts.Apply(loc.Scheme, &cfg); err != nil { - return nil, err - } - - debug.Log("opening gs repository at %#v", cfg) - return cfg, nil - - case "swift": - cfg := loc.Config.(swift.Config) - - if err := swift.ApplyEnvironment("", &cfg); err != nil { - return nil, err - } - - if err := opts.Apply(loc.Scheme, &cfg); err != nil { - return nil, err - } - - debug.Log("opening swift repository at %#v", cfg) - return cfg, nil - - case "b2": - cfg := loc.Config.(b2.Config) - - if cfg.AccountID == "" { - cfg.AccountID = os.Getenv("B2_ACCOUNT_ID") - } - - if cfg.AccountID == "" { - return nil, errors.Fatalf("unable to open B2 backend: Account ID ($B2_ACCOUNT_ID) is empty") - } - - if cfg.Key.String() == "" { - cfg.Key = options.NewSecretString(os.Getenv("B2_ACCOUNT_KEY")) - } - - if cfg.Key.String() == "" { - return nil, errors.Fatalf("unable to open B2 backend: Key ($B2_ACCOUNT_KEY) is empty") - } - - if err := opts.Apply(loc.Scheme, &cfg); err != nil { - return nil, err - } - - debug.Log("opening b2 repository at %#v", cfg) - return cfg, nil - case "rest": - cfg := loc.Config.(rest.Config) - if err := opts.Apply(loc.Scheme, &cfg); err != nil { - return nil, err - } - - debug.Log("opening rest repository at %#v", cfg) - return cfg, nil - case "rclone": - cfg := loc.Config.(rclone.Config) - if err := opts.Apply(loc.Scheme, &cfg); err != nil { - return nil, err - } - - debug.Log("opening rest repository at %#v", cfg) - return cfg, nil - case "smb": - cfg := loc.Config.(smb.Config) - if cfg.User == "" { - cfg.User = os.Getenv("RESTIC_SMB_USER") - } - - if cfg.Password.String() == "" { - cfg.Password = options.NewSecretString(os.Getenv("RESTIC_SMB_PASSWORD")) - } - - if cfg.Domain == "" { - cfg.Domain = os.Getenv("RESTIC_SMB_DOMAIN") - } - if cfg.Domain == "" { - cfg.Domain = smb.DefaultDomain - } - if err := opts.Apply(loc.Scheme, &cfg); err != nil { - return nil, err - } - - debug.Log("opening smb repository at %#v", cfg) - return cfg, nil - } - return nil, errors.Fatalf("invalid backend: %q", loc.Scheme) + // only apply options for a particular backend here + opts = opts.Extract(loc.Scheme) + if err := opts.Apply(loc.Scheme, cfg); err != nil { + return nil, err + } + + debug.Log("opening %v repository at %#v", loc.Scheme, cfg) + return cfg, nil } // Open the backend specified by a location config. @@ -728,25 +579,25 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio switch loc.Scheme { case "local": - be, err = local.Open(ctx, cfg.(local.Config)) + be, err = local.Open(ctx, *cfg.(*local.Config)) case "sftp": - be, err = sftp.Open(ctx, cfg.(sftp.Config)) - case "smb": - be, err = smb.Open(ctx, cfg.(smb.Config)) + be, err = sftp.Open(ctx, *cfg.(*sftp.Config)) case "s3": - be, err = s3.Open(ctx, cfg.(s3.Config), rt) + be, err = s3.Open(ctx, *cfg.(*s3.Config), rt) case "gs": - be, err = gs.Open(cfg.(gs.Config), rt) + be, err = gs.Open(*cfg.(*gs.Config), rt) case "azure": - be, err = azure.Open(ctx, cfg.(azure.Config), rt) + be, err = azure.Open(ctx, *cfg.(*azure.Config), rt) case "swift": - be, err = swift.Open(ctx, cfg.(swift.Config), rt) + be, err = swift.Open(ctx, *cfg.(*swift.Config), rt) case "b2": - be, err = b2.Open(ctx, cfg.(b2.Config), rt) + be, err = b2.Open(ctx, *cfg.(*b2.Config), rt) case "rest": - be, err = rest.Open(cfg.(rest.Config), rt) + be, err = rest.Open(*cfg.(*rest.Config), rt) case "rclone": - be, err = rclone.Open(cfg.(rclone.Config), lim) + be, err = rclone.Open(*cfg.(*rclone.Config), lim) + case "smb": + be, err = smb.Open(ctx, *cfg.(*smb.Config)) default: return nil, errors.Fatalf("invalid backend: %q", loc.Scheme) @@ -806,25 +657,25 @@ func create(ctx context.Context, s string, opts options.Options) (restic.Backend var be restic.Backend switch loc.Scheme { case "local": - be, err = local.Create(ctx, cfg.(local.Config)) + be, err = local.Create(ctx, *cfg.(*local.Config)) case "sftp": - be, err = sftp.Create(ctx, cfg.(sftp.Config)) - case "smb": - be, err = smb.Create(ctx, cfg.(smb.Config)) + be, err = sftp.Create(ctx, *cfg.(*sftp.Config)) case "s3": - be, err = s3.Create(ctx, cfg.(s3.Config), rt) + be, err = s3.Create(ctx, *cfg.(*s3.Config), rt) case "gs": - be, err = gs.Create(ctx, cfg.(gs.Config), rt) + be, err = gs.Create(ctx, *cfg.(*gs.Config), rt) case "azure": - be, err = azure.Create(ctx, cfg.(azure.Config), rt) + be, err = azure.Create(ctx, *cfg.(*azure.Config), rt) case "swift": - be, err = swift.Open(ctx, cfg.(swift.Config), rt) + be, err = swift.Open(ctx, *cfg.(*swift.Config), rt) case "b2": - be, err = b2.Create(ctx, cfg.(b2.Config), rt) + be, err = b2.Create(ctx, *cfg.(*b2.Config), rt) case "rest": - be, err = rest.Create(ctx, cfg.(rest.Config), rt) + be, err = rest.Create(ctx, *cfg.(*rest.Config), rt) case "rclone": - be, err = rclone.Create(ctx, cfg.(rclone.Config)) + be, err = rclone.Create(ctx, *cfg.(*rclone.Config)) + case "smb": + be, err = smb.Create(ctx, *cfg.(*smb.Config)) default: debug.Log("invalid repository scheme: %v", s) return nil, errors.Fatalf("invalid scheme %q", loc.Scheme) diff --git a/internal/backend/azure/azure_test.go b/internal/backend/azure/azure_test.go index ada6ec2ca..0fab5da26 100644 --- a/internal/backend/azure/azure_test.go +++ b/internal/backend/azure/azure_test.go @@ -18,34 +18,34 @@ import ( rtest "github.com/restic/restic/internal/test" ) -func newAzureTestSuite(t testing.TB) *test.Suite { +func newAzureTestSuite(t testing.TB) *test.Suite[azure.Config] { tr, err := backend.Transport(backend.TransportOptions{}) if err != nil { t.Fatalf("cannot create transport for tests: %v", err) } - return &test.Suite{ + return &test.Suite[azure.Config]{ // do not use excessive data MinimalData: true, // NewConfig returns a config for a new temporary backend that will be used in tests. - NewConfig: func() (interface{}, error) { - azcfg, err := azure.ParseConfig(os.Getenv("RESTIC_TEST_AZURE_REPOSITORY")) + NewConfig: func() (*azure.Config, error) { + cfg, err := azure.ParseConfig(os.Getenv("RESTIC_TEST_AZURE_REPOSITORY")) + if err != nil { + return nil, err + } + + err = cfg.ApplyEnvironment("RESTIC_TEST_") if err != nil { return nil, err } - cfg := azcfg.(azure.Config) - cfg.AccountName = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_NAME") - cfg.AccountKey = options.NewSecretString(os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY")) cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) return cfg, nil }, // CreateFn is a function that creates a temporary repository for the tests. - Create: func(config interface{}) (restic.Backend, error) { - cfg := config.(azure.Config) - + Create: func(cfg azure.Config) (restic.Backend, error) { ctx := context.TODO() be, err := azure.Create(ctx, cfg, tr) if err != nil { @@ -65,15 +65,13 @@ func newAzureTestSuite(t testing.TB) *test.Suite { }, // OpenFn is a function that opens a previously created temporary repository. - Open: func(config interface{}) (restic.Backend, error) { - cfg := config.(azure.Config) + Open: func(cfg azure.Config) (restic.Backend, error) { ctx := context.TODO() return azure.Open(ctx, cfg, tr) }, // CleanupFn removes data created during the tests. - Cleanup: func(config interface{}) error { - cfg := config.(azure.Config) + Cleanup: func(cfg azure.Config) error { ctx := context.TODO() be, err := azure.Open(ctx, cfg, tr) if err != nil { @@ -141,12 +139,11 @@ func TestUploadLargeFile(t *testing.T) { return } - azcfg, err := azure.ParseConfig(os.Getenv("RESTIC_TEST_AZURE_REPOSITORY")) + cfg, err := azure.ParseConfig(os.Getenv("RESTIC_TEST_AZURE_REPOSITORY")) if err != nil { t.Fatal(err) } - cfg := azcfg.(azure.Config) cfg.AccountName = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_NAME") cfg.AccountKey = options.NewSecretString(os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY")) cfg.Prefix = fmt.Sprintf("test-upload-large-%d", time.Now().UnixNano()) @@ -156,7 +153,7 @@ func TestUploadLargeFile(t *testing.T) { t.Fatal(err) } - be, err := azure.Create(ctx, cfg, tr) + be, err := azure.Create(ctx, *cfg, tr) if err != nil { t.Fatal(err) } diff --git a/internal/backend/azure/config.go b/internal/backend/azure/config.go index 55b26d4f1..4d4e839ff 100644 --- a/internal/backend/azure/config.go +++ b/internal/backend/azure/config.go @@ -1,11 +1,13 @@ package azure import ( + "os" "path" "strings" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" + "github.com/restic/restic/internal/restic" ) // Config contains all configuration necessary to connect to an azure compatible @@ -33,7 +35,7 @@ func init() { // ParseConfig parses the string s and extracts the azure config. The // configuration format is azure:containerName:/[prefix]. -func ParseConfig(s string) (interface{}, error) { +func ParseConfig(s string) (*Config, error) { if !strings.HasPrefix(s, "azure:") { return nil, errors.New("azure: invalid format") } @@ -51,5 +53,23 @@ func ParseConfig(s string) (interface{}, error) { cfg := NewConfig() cfg.Container = container cfg.Prefix = prefix - return cfg, nil + return &cfg, nil +} + +var _ restic.ApplyEnvironmenter = &Config{} + +// ApplyEnvironment saves values from the environment to the config. +func (cfg *Config) ApplyEnvironment(prefix string) error { + if cfg.AccountName == "" { + cfg.AccountName = os.Getenv(prefix + "AZURE_ACCOUNT_NAME") + } + + if cfg.AccountKey.String() == "" { + cfg.AccountKey = options.NewSecretString(os.Getenv(prefix + "AZURE_ACCOUNT_KEY")) + } + + if cfg.AccountSAS.String() == "" { + cfg.AccountSAS = options.NewSecretString(os.Getenv(prefix + "AZURE_ACCOUNT_SAS")) + } + return nil } diff --git a/internal/backend/azure/config_test.go b/internal/backend/azure/config_test.go index a57542e77..49cda6571 100644 --- a/internal/backend/azure/config_test.go +++ b/internal/backend/azure/config_test.go @@ -1,22 +1,23 @@ package azure -import "testing" +import ( + "testing" -var configTests = []struct { - s string - cfg Config -}{ - {"azure:container-name:/", Config{ + "github.com/restic/restic/internal/backend/test" +) + +var configTests = []test.ConfigTestData[Config]{ + {S: "azure:container-name:/", Cfg: Config{ Container: "container-name", Prefix: "", Connections: 5, }}, - {"azure:container-name:/prefix/directory", Config{ + {S: "azure:container-name:/prefix/directory", Cfg: Config{ Container: "container-name", Prefix: "prefix/directory", Connections: 5, }}, - {"azure:container-name:/prefix/directory/", Config{ + {S: "azure:container-name:/prefix/directory/", Cfg: Config{ Container: "container-name", Prefix: "prefix/directory", Connections: 5, @@ -24,17 +25,5 @@ var configTests = []struct { } func TestParseConfig(t *testing.T) { - for i, test := range configTests { - cfg, err := ParseConfig(test.s) - if err != nil { - t.Errorf("test %d:%s failed: %v", i, test.s, err) - continue - } - - if cfg != test.cfg { - t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v", - i, test.s, test.cfg, cfg) - continue - } - } + test.ParseConfigTester(t, ParseConfig, configTests) } diff --git a/internal/backend/b2/b2_test.go b/internal/backend/b2/b2_test.go index 123a61d7c..8e982adda 100644 --- a/internal/backend/b2/b2_test.go +++ b/internal/backend/b2/b2_test.go @@ -10,19 +10,18 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/b2" "github.com/restic/restic/internal/backend/test" - "github.com/restic/restic/internal/options" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) -func newB2TestSuite(t testing.TB) *test.Suite { +func newB2TestSuite(t testing.TB) *test.Suite[b2.Config] { tr, err := backend.Transport(backend.TransportOptions{}) if err != nil { t.Fatalf("cannot create transport for tests: %v", err) } - return &test.Suite{ + return &test.Suite[b2.Config]{ // do not use excessive data MinimalData: true, @@ -30,34 +29,33 @@ func newB2TestSuite(t testing.TB) *test.Suite { WaitForDelayedRemoval: 10 * time.Second, // NewConfig returns a config for a new temporary backend that will be used in tests. - NewConfig: func() (interface{}, error) { - b2cfg, err := b2.ParseConfig(os.Getenv("RESTIC_TEST_B2_REPOSITORY")) + NewConfig: func() (*b2.Config, error) { + cfg, err := b2.ParseConfig(os.Getenv("RESTIC_TEST_B2_REPOSITORY")) + if err != nil { + return nil, err + } + + err = cfg.ApplyEnvironment("RESTIC_TEST_") if err != nil { return nil, err } - cfg := b2cfg.(b2.Config) - cfg.AccountID = os.Getenv("RESTIC_TEST_B2_ACCOUNT_ID") - cfg.Key = options.NewSecretString(os.Getenv("RESTIC_TEST_B2_ACCOUNT_KEY")) cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) return cfg, nil }, // CreateFn is a function that creates a temporary repository for the tests. - Create: func(config interface{}) (restic.Backend, error) { - cfg := config.(b2.Config) + Create: func(cfg b2.Config) (restic.Backend, error) { return b2.Create(context.Background(), cfg, tr) }, // OpenFn is a function that opens a previously created temporary repository. - Open: func(config interface{}) (restic.Backend, error) { - cfg := config.(b2.Config) + Open: func(cfg b2.Config) (restic.Backend, error) { return b2.Open(context.Background(), cfg, tr) }, // CleanupFn removes data created during the tests. - Cleanup: func(config interface{}) error { - cfg := config.(b2.Config) + Cleanup: func(cfg b2.Config) error { be, err := b2.Open(context.Background(), cfg, tr) if err != nil { return err diff --git a/internal/backend/b2/config.go b/internal/backend/b2/config.go index ba5141834..548fbef99 100644 --- a/internal/backend/b2/config.go +++ b/internal/backend/b2/config.go @@ -1,12 +1,14 @@ package b2 import ( + "os" "path" "regexp" "strings" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" + "github.com/restic/restic/internal/restic" ) // Config contains all configuration necessary to connect to an b2 compatible @@ -58,7 +60,7 @@ func checkBucketName(name string) error { // ParseConfig parses the string s and extracts the b2 config. The supported // configuration format is b2:bucketname/prefix. If no prefix is given the // prefix "restic" will be used. -func ParseConfig(s string) (interface{}, error) { +func ParseConfig(s string) (*Config, error) { if !strings.HasPrefix(s, "b2:") { return nil, errors.New("invalid format, want: b2:bucket-name[:path]") } @@ -77,5 +79,27 @@ func ParseConfig(s string) (interface{}, error) { cfg.Bucket = bucket cfg.Prefix = prefix - return cfg, nil + return &cfg, nil +} + +var _ restic.ApplyEnvironmenter = &Config{} + +// ApplyEnvironment saves values from the environment to the config. +func (cfg *Config) ApplyEnvironment(prefix string) error { + if cfg.AccountID == "" { + cfg.AccountID = os.Getenv(prefix + "B2_ACCOUNT_ID") + } + + if cfg.AccountID == "" { + return errors.Fatalf("unable to open B2 backend: Account ID ($B2_ACCOUNT_ID) is empty") + } + + if cfg.Key.String() == "" { + cfg.Key = options.NewSecretString(os.Getenv(prefix + "B2_ACCOUNT_KEY")) + } + + if cfg.Key.String() == "" { + return errors.Fatalf("unable to open B2 backend: Key ($B2_ACCOUNT_KEY) is empty") + } + return nil } diff --git a/internal/backend/b2/config_test.go b/internal/backend/b2/config_test.go index 4194cb62c..f62972005 100644 --- a/internal/backend/b2/config_test.go +++ b/internal/backend/b2/config_test.go @@ -1,37 +1,38 @@ package b2 -import "testing" +import ( + "testing" -var configTests = []struct { - s string - cfg Config -}{ - {"b2:bucketname", Config{ + "github.com/restic/restic/internal/backend/test" +) + +var configTests = []test.ConfigTestData[Config]{ + {S: "b2:bucketname", Cfg: Config{ Bucket: "bucketname", Prefix: "", Connections: 5, }}, - {"b2:bucketname:", Config{ + {S: "b2:bucketname:", Cfg: Config{ Bucket: "bucketname", Prefix: "", Connections: 5, }}, - {"b2:bucketname:/prefix/directory", Config{ + {S: "b2:bucketname:/prefix/directory", Cfg: Config{ Bucket: "bucketname", Prefix: "prefix/directory", Connections: 5, }}, - {"b2:foobar", Config{ + {S: "b2:foobar", Cfg: Config{ Bucket: "foobar", Prefix: "", Connections: 5, }}, - {"b2:foobar:", Config{ + {S: "b2:foobar:", Cfg: Config{ Bucket: "foobar", Prefix: "", Connections: 5, }}, - {"b2:foobar:/", Config{ + {S: "b2:foobar:/", Cfg: Config{ Bucket: "foobar", Prefix: "", Connections: 5, @@ -39,19 +40,7 @@ var configTests = []struct { } func TestParseConfig(t *testing.T) { - for _, test := range configTests { - t.Run("", func(t *testing.T) { - cfg, err := ParseConfig(test.s) - if err != nil { - t.Fatalf("%s failed: %v", test.s, err) - } - - if cfg != test.cfg { - t.Fatalf("input: %s\n wrong config, want:\n %#v\ngot:\n %#v", - test.s, test.cfg, cfg) - } - }) - } + test.ParseConfigTester(t, ParseConfig, configTests) } var invalidConfigTests = []struct { diff --git a/internal/backend/gs/config.go b/internal/backend/gs/config.go index 8bb2bddea..b2d52c5f8 100644 --- a/internal/backend/gs/config.go +++ b/internal/backend/gs/config.go @@ -1,11 +1,13 @@ package gs import ( + "os" "path" "strings" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" + "github.com/restic/restic/internal/restic" ) // Config contains all configuration necessary to connect to a Google Cloud Storage @@ -34,7 +36,7 @@ func init() { // ParseConfig parses the string s and extracts the gcs config. The // supported configuration format is gs:bucketName:/[prefix]. -func ParseConfig(s string) (interface{}, error) { +func ParseConfig(s string) (*Config, error) { if !strings.HasPrefix(s, "gs:") { return nil, errors.New("gs: invalid format") } @@ -54,5 +56,15 @@ func ParseConfig(s string) (interface{}, error) { cfg := NewConfig() cfg.Bucket = bucket cfg.Prefix = prefix - return cfg, nil + return &cfg, nil +} + +var _ restic.ApplyEnvironmenter = &Config{} + +// ApplyEnvironment saves values from the environment to the config. +func (cfg *Config) ApplyEnvironment(prefix string) error { + if cfg.ProjectID == "" { + cfg.ProjectID = os.Getenv(prefix + "GOOGLE_PROJECT_ID") + } + return nil } diff --git a/internal/backend/gs/config_test.go b/internal/backend/gs/config_test.go index fb2774d25..890de577f 100644 --- a/internal/backend/gs/config_test.go +++ b/internal/backend/gs/config_test.go @@ -1,24 +1,25 @@ package gs -import "testing" +import ( + "testing" -var configTests = []struct { - s string - cfg Config -}{ - {"gs:bucketname:/", Config{ + "github.com/restic/restic/internal/backend/test" +) + +var configTests = []test.ConfigTestData[Config]{ + {S: "gs:bucketname:/", Cfg: Config{ Bucket: "bucketname", Prefix: "", Connections: 5, Region: "us", }}, - {"gs:bucketname:/prefix/directory", Config{ + {S: "gs:bucketname:/prefix/directory", Cfg: Config{ Bucket: "bucketname", Prefix: "prefix/directory", Connections: 5, Region: "us", }}, - {"gs:bucketname:/prefix/directory/", Config{ + {S: "gs:bucketname:/prefix/directory/", Cfg: Config{ Bucket: "bucketname", Prefix: "prefix/directory", Connections: 5, @@ -27,17 +28,5 @@ var configTests = []struct { } func TestParseConfig(t *testing.T) { - for i, test := range configTests { - cfg, err := ParseConfig(test.s) - if err != nil { - t.Errorf("test %d:%s failed: %v", i, test.s, err) - continue - } - - if cfg != test.cfg { - t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v", - i, test.s, test.cfg, cfg) - continue - } - } + test.ParseConfigTester(t, ParseConfig, configTests) } diff --git a/internal/backend/gs/gs_test.go b/internal/backend/gs/gs_test.go index 19ae8b829..f96b6c62b 100644 --- a/internal/backend/gs/gs_test.go +++ b/internal/backend/gs/gs_test.go @@ -15,33 +15,30 @@ import ( rtest "github.com/restic/restic/internal/test" ) -func newGSTestSuite(t testing.TB) *test.Suite { +func newGSTestSuite(t testing.TB) *test.Suite[gs.Config] { tr, err := backend.Transport(backend.TransportOptions{}) if err != nil { t.Fatalf("cannot create transport for tests: %v", err) } - return &test.Suite{ + return &test.Suite[gs.Config]{ // do not use excessive data MinimalData: true, // NewConfig returns a config for a new temporary backend that will be used in tests. - NewConfig: func() (interface{}, error) { - gscfg, err := gs.ParseConfig(os.Getenv("RESTIC_TEST_GS_REPOSITORY")) + NewConfig: func() (*gs.Config, error) { + cfg, err := gs.ParseConfig(os.Getenv("RESTIC_TEST_GS_REPOSITORY")) if err != nil { return nil, err } - cfg := gscfg.(gs.Config) cfg.ProjectID = os.Getenv("RESTIC_TEST_GS_PROJECT_ID") cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) return cfg, nil }, // CreateFn is a function that creates a temporary repository for the tests. - Create: func(config interface{}) (restic.Backend, error) { - cfg := config.(gs.Config) - + Create: func(cfg gs.Config) (restic.Backend, error) { be, err := gs.Create(context.Background(), cfg, tr) if err != nil { return nil, err @@ -60,15 +57,12 @@ func newGSTestSuite(t testing.TB) *test.Suite { }, // OpenFn is a function that opens a previously created temporary repository. - Open: func(config interface{}) (restic.Backend, error) { - cfg := config.(gs.Config) + Open: func(cfg gs.Config) (restic.Backend, error) { return gs.Open(cfg, tr) }, // CleanupFn removes data created during the tests. - Cleanup: func(config interface{}) error { - cfg := config.(gs.Config) - + Cleanup: func(cfg gs.Config) error { be, err := gs.Open(cfg, tr) if err != nil { return err diff --git a/internal/backend/local/config.go b/internal/backend/local/config.go index e59d1f693..dc5e7948c 100644 --- a/internal/backend/local/config.go +++ b/internal/backend/local/config.go @@ -27,12 +27,12 @@ func init() { } // ParseConfig parses a local backend config. -func ParseConfig(s string) (interface{}, error) { +func ParseConfig(s string) (*Config, error) { if !strings.HasPrefix(s, "local:") { return nil, errors.New(`invalid format, prefix "local" not found`) } cfg := NewConfig() cfg.Path = s[6:] - return cfg, nil + return &cfg, nil } diff --git a/internal/backend/local/config_test.go b/internal/backend/local/config_test.go new file mode 100644 index 000000000..c9b6be61c --- /dev/null +++ b/internal/backend/local/config_test.go @@ -0,0 +1,18 @@ +package local + +import ( + "testing" + + "github.com/restic/restic/internal/backend/test" +) + +var configTests = []test.ConfigTestData[Config]{ + {S: "local:/some/path", Cfg: Config{ + Path: "/some/path", + Connections: 2, + }}, +} + +func TestParseConfig(t *testing.T) { + test.ParseConfigTester(t, ParseConfig, configTests) +} diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index be332e8ad..d6bdef1e4 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -148,6 +148,13 @@ func (b *Local) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) } }(f) + // preallocate disk space + if size := rd.Length(); size > 0 { + if err := fs.PreallocateFile(f, size); err != nil { + debug.Log("Failed to preallocate %v with size %v: %v", finalname, size, err) + } + } + // save data, then sync wbytes, err := io.Copy(f, rd) if err != nil { diff --git a/internal/backend/local/local_test.go b/internal/backend/local/local_test.go index 495f220a0..ca9e3b71b 100644 --- a/internal/backend/local/local_test.go +++ b/internal/backend/local/local_test.go @@ -12,10 +12,10 @@ import ( rtest "github.com/restic/restic/internal/test" ) -func newTestSuite(t testing.TB) *test.Suite { - return &test.Suite{ +func newTestSuite(t testing.TB) *test.Suite[local.Config] { + return &test.Suite[local.Config]{ // NewConfig returns a config for a new temporary backend that will be used in tests. - NewConfig: func() (interface{}, error) { + NewConfig: func() (*local.Config, error) { dir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-local-") if err != nil { t.Fatal(err) @@ -23,7 +23,7 @@ func newTestSuite(t testing.TB) *test.Suite { t.Logf("create new backend at %v", dir) - cfg := local.Config{ + cfg := &local.Config{ Path: dir, Connections: 2, } @@ -31,20 +31,17 @@ func newTestSuite(t testing.TB) *test.Suite { }, // CreateFn is a function that creates a temporary repository for the tests. - Create: func(config interface{}) (restic.Backend, error) { - cfg := config.(local.Config) + Create: func(cfg local.Config) (restic.Backend, error) { return local.Create(context.TODO(), cfg) }, // OpenFn is a function that opens a previously created temporary repository. - Open: func(config interface{}) (restic.Backend, error) { - cfg := config.(local.Config) + Open: func(cfg local.Config) (restic.Backend, error) { return local.Open(context.TODO(), cfg) }, // CleanupFn removes data created during the tests. - Cleanup: func(config interface{}) error { - cfg := config.(local.Config) + Cleanup: func(cfg local.Config) error { if !rtest.TestCleanupTempDirs { t.Logf("leaving test backend dir at %v", cfg.Path) } diff --git a/internal/backend/location/location.go b/internal/backend/location/location.go index c9bbf981f..dcc748744 100644 --- a/internal/backend/location/location.go +++ b/internal/backend/location/location.go @@ -30,19 +30,25 @@ type parser struct { stripPassword func(string) string } +func configToAny[C any](parser func(string) (*C, error)) func(string) (interface{}, error) { + return func(s string) (interface{}, error) { + return parser(s) + } +} + // parsers is a list of valid config parsers for the backends. The first parser // is the fallback and should always be set to the local backend. var parsers = []parser{ - {"b2", b2.ParseConfig, noPassword}, - {"local", local.ParseConfig, noPassword}, - {"sftp", sftp.ParseConfig, noPassword}, - {"s3", s3.ParseConfig, noPassword}, - {"gs", gs.ParseConfig, noPassword}, - {"azure", azure.ParseConfig, noPassword}, - {"swift", swift.ParseConfig, noPassword}, - {"rest", rest.ParseConfig, rest.StripPassword}, - {"rclone", rclone.ParseConfig, noPassword}, - {"smb", smb.ParseConfig, noPassword}, + {"b2", configToAny(b2.ParseConfig), noPassword}, + {"local", configToAny(local.ParseConfig), noPassword}, + {"sftp", configToAny(sftp.ParseConfig), noPassword}, + {"s3", configToAny(s3.ParseConfig), noPassword}, + {"gs", configToAny(gs.ParseConfig), noPassword}, + {"azure", configToAny(azure.ParseConfig), noPassword}, + {"swift", configToAny(swift.ParseConfig), noPassword}, + {"rest", configToAny(rest.ParseConfig), rest.StripPassword}, + {"rclone", configToAny(rclone.ParseConfig), noPassword}, + {"smb", configToAny(smb.ParseConfig), noPassword}, } // noPassword returns the repository location unchanged (there's no sensitive information there) diff --git a/internal/backend/location/location_test.go b/internal/backend/location/location_test.go index 809379850..9f5db70c9 100644 --- a/internal/backend/location/location_test.go +++ b/internal/backend/location/location_test.go @@ -29,7 +29,7 @@ var parseTests = []struct { { "local:/srv/repo", Location{Scheme: "local", - Config: local.Config{ + Config: &local.Config{ Path: "/srv/repo", Connections: 2, }, @@ -38,7 +38,7 @@ var parseTests = []struct { { "local:dir1/dir2", Location{Scheme: "local", - Config: local.Config{ + Config: &local.Config{ Path: "dir1/dir2", Connections: 2, }, @@ -47,7 +47,7 @@ var parseTests = []struct { { "local:dir1/dir2", Location{Scheme: "local", - Config: local.Config{ + Config: &local.Config{ Path: "dir1/dir2", Connections: 2, }, @@ -56,7 +56,7 @@ var parseTests = []struct { { "dir1/dir2", Location{Scheme: "local", - Config: local.Config{ + Config: &local.Config{ Path: "dir1/dir2", Connections: 2, }, @@ -65,7 +65,7 @@ var parseTests = []struct { { "/dir1/dir2", Location{Scheme: "local", - Config: local.Config{ + Config: &local.Config{ Path: "/dir1/dir2", Connections: 2, }, @@ -74,7 +74,7 @@ var parseTests = []struct { { "local:../dir1/dir2", Location{Scheme: "local", - Config: local.Config{ + Config: &local.Config{ Path: "../dir1/dir2", Connections: 2, }, @@ -83,7 +83,7 @@ var parseTests = []struct { { "/dir1/dir2", Location{Scheme: "local", - Config: local.Config{ + Config: &local.Config{ Path: "/dir1/dir2", Connections: 2, }, @@ -92,7 +92,7 @@ var parseTests = []struct { { "/dir1:foobar/dir2", Location{Scheme: "local", - Config: local.Config{ + Config: &local.Config{ Path: "/dir1:foobar/dir2", Connections: 2, }, @@ -101,7 +101,7 @@ var parseTests = []struct { { `\dir1\foobar\dir2`, Location{Scheme: "local", - Config: local.Config{ + Config: &local.Config{ Path: `\dir1\foobar\dir2`, Connections: 2, }, @@ -110,7 +110,7 @@ var parseTests = []struct { { `c:\dir1\foobar\dir2`, Location{Scheme: "local", - Config: local.Config{ + Config: &local.Config{ Path: `c:\dir1\foobar\dir2`, Connections: 2, }, @@ -119,7 +119,7 @@ var parseTests = []struct { { `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`, Location{Scheme: "local", - Config: local.Config{ + Config: &local.Config{ Path: `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`, Connections: 2, }, @@ -128,7 +128,7 @@ var parseTests = []struct { { `c:/dir1/foobar/dir2`, Location{Scheme: "local", - Config: local.Config{ + Config: &local.Config{ Path: `c:/dir1/foobar/dir2`, Connections: 2, }, @@ -137,7 +137,7 @@ var parseTests = []struct { { "sftp:user@host:/srv/repo", Location{Scheme: "sftp", - Config: sftp.Config{ + Config: &sftp.Config{ User: "user", Host: "host", Path: "/srv/repo", @@ -148,7 +148,7 @@ var parseTests = []struct { { "sftp:host:/srv/repo", Location{Scheme: "sftp", - Config: sftp.Config{ + Config: &sftp.Config{ User: "", Host: "host", Path: "/srv/repo", @@ -159,7 +159,7 @@ var parseTests = []struct { { "sftp://user@host/srv/repo", Location{Scheme: "sftp", - Config: sftp.Config{ + Config: &sftp.Config{ User: "user", Host: "host", Path: "srv/repo", @@ -170,7 +170,7 @@ var parseTests = []struct { { "sftp://user@host//srv/repo", Location{Scheme: "sftp", - Config: sftp.Config{ + Config: &sftp.Config{ User: "user", Host: "host", Path: "/srv/repo", @@ -182,7 +182,7 @@ var parseTests = []struct { { "s3://eu-central-1/bucketname", Location{Scheme: "s3", - Config: s3.Config{ + Config: &s3.Config{ Endpoint: "eu-central-1", Bucket: "bucketname", Prefix: "", @@ -193,7 +193,7 @@ var parseTests = []struct { { "s3://hostname.foo/bucketname", Location{Scheme: "s3", - Config: s3.Config{ + Config: &s3.Config{ Endpoint: "hostname.foo", Bucket: "bucketname", Prefix: "", @@ -204,7 +204,7 @@ var parseTests = []struct { { "s3://hostname.foo/bucketname/prefix/directory", Location{Scheme: "s3", - Config: s3.Config{ + Config: &s3.Config{ Endpoint: "hostname.foo", Bucket: "bucketname", Prefix: "prefix/directory", @@ -215,7 +215,7 @@ var parseTests = []struct { { "s3:eu-central-1/repo", Location{Scheme: "s3", - Config: s3.Config{ + Config: &s3.Config{ Endpoint: "eu-central-1", Bucket: "repo", Prefix: "", @@ -226,7 +226,7 @@ var parseTests = []struct { { "s3:eu-central-1/repo/prefix/directory", Location{Scheme: "s3", - Config: s3.Config{ + Config: &s3.Config{ Endpoint: "eu-central-1", Bucket: "repo", Prefix: "prefix/directory", @@ -237,7 +237,7 @@ var parseTests = []struct { { "s3:https://hostname.foo/repo", Location{Scheme: "s3", - Config: s3.Config{ + Config: &s3.Config{ Endpoint: "hostname.foo", Bucket: "repo", Prefix: "", @@ -248,7 +248,7 @@ var parseTests = []struct { { "s3:https://hostname.foo/repo/prefix/directory", Location{Scheme: "s3", - Config: s3.Config{ + Config: &s3.Config{ Endpoint: "hostname.foo", Bucket: "repo", Prefix: "prefix/directory", @@ -259,7 +259,7 @@ var parseTests = []struct { { "s3:http://hostname.foo/repo", Location{Scheme: "s3", - Config: s3.Config{ + Config: &s3.Config{ Endpoint: "hostname.foo", Bucket: "repo", Prefix: "", @@ -271,7 +271,7 @@ var parseTests = []struct { { "swift:container17:/", Location{Scheme: "swift", - Config: swift.Config{ + Config: &swift.Config{ Container: "container17", Prefix: "", Connections: 5, @@ -281,7 +281,7 @@ var parseTests = []struct { { "swift:container17:/prefix97", Location{Scheme: "swift", - Config: swift.Config{ + Config: &swift.Config{ Container: "container17", Prefix: "prefix97", Connections: 5, @@ -291,7 +291,7 @@ var parseTests = []struct { { "rest:http://hostname.foo:1234/", Location{Scheme: "rest", - Config: rest.Config{ + Config: &rest.Config{ URL: parseURL("http://hostname.foo:1234/"), Connections: 5, }, @@ -299,7 +299,7 @@ var parseTests = []struct { }, { "b2:bucketname:/prefix", Location{Scheme: "b2", - Config: b2.Config{ + Config: &b2.Config{ Bucket: "bucketname", Prefix: "prefix", Connections: 5, @@ -308,7 +308,7 @@ var parseTests = []struct { }, { "b2:bucketname", Location{Scheme: "b2", - Config: b2.Config{ + Config: &b2.Config{ Bucket: "bucketname", Prefix: "", Connections: 5, diff --git a/internal/backend/mem/mem_backend_test.go b/internal/backend/mem/mem_backend_test.go index 819c6a2b6..3dea089bc 100644 --- a/internal/backend/mem/mem_backend_test.go +++ b/internal/backend/mem/mem_backend_test.go @@ -15,19 +15,19 @@ type memConfig struct { be restic.Backend } -func newTestSuite() *test.Suite { - return &test.Suite{ +func newTestSuite() *test.Suite[*memConfig] { + return &test.Suite[*memConfig]{ // NewConfig returns a config for a new temporary backend that will be used in tests. - NewConfig: func() (interface{}, error) { - return &memConfig{}, nil + NewConfig: func() (**memConfig, error) { + cfg := &memConfig{} + return &cfg, nil }, // CreateFn is a function that creates a temporary repository for the tests. - Create: func(cfg interface{}) (restic.Backend, error) { - c := cfg.(*memConfig) - if c.be != nil { - _, err := c.be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile}) - if err != nil && !c.be.IsNotExist(err) { + Create: func(cfg *memConfig) (restic.Backend, error) { + if cfg.be != nil { + _, err := cfg.be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile}) + if err != nil && !cfg.be.IsNotExist(err) { return nil, err } @@ -36,21 +36,20 @@ func newTestSuite() *test.Suite { } } - c.be = mem.New() - return c.be, nil + cfg.be = mem.New() + return cfg.be, nil }, // OpenFn is a function that opens a previously created temporary repository. - Open: func(cfg interface{}) (restic.Backend, error) { - c := cfg.(*memConfig) - if c.be == nil { - c.be = mem.New() + Open: func(cfg *memConfig) (restic.Backend, error) { + if cfg.be == nil { + cfg.be = mem.New() } - return c.be, nil + return cfg.be, nil }, // CleanupFn removes data created during the tests. - Cleanup: func(cfg interface{}) error { + Cleanup: func(cfg *memConfig) error { // no cleanup needed return nil }, diff --git a/internal/backend/rclone/backend_test.go b/internal/backend/rclone/backend_test.go index 12fed6274..c497271f6 100644 --- a/internal/backend/rclone/backend_test.go +++ b/internal/backend/rclone/backend_test.go @@ -12,22 +12,21 @@ import ( rtest "github.com/restic/restic/internal/test" ) -func newTestSuite(t testing.TB) *test.Suite { +func newTestSuite(t testing.TB) *test.Suite[rclone.Config] { dir := rtest.TempDir(t) - return &test.Suite{ + return &test.Suite[rclone.Config]{ // NewConfig returns a config for a new temporary backend that will be used in tests. - NewConfig: func() (interface{}, error) { + NewConfig: func() (*rclone.Config, error) { t.Logf("use backend at %v", dir) cfg := rclone.NewConfig() cfg.Remote = dir - return cfg, nil + return &cfg, nil }, // CreateFn is a function that creates a temporary repository for the tests. - Create: func(config interface{}) (restic.Backend, error) { + Create: func(cfg rclone.Config) (restic.Backend, error) { t.Logf("Create()") - cfg := config.(rclone.Config) be, err := rclone.Create(context.TODO(), cfg) var e *exec.Error if errors.As(err, &e) && e.Err == exec.ErrNotFound { @@ -38,9 +37,8 @@ func newTestSuite(t testing.TB) *test.Suite { }, // OpenFn is a function that opens a previously created temporary repository. - Open: func(config interface{}) (restic.Backend, error) { + Open: func(cfg rclone.Config) (restic.Backend, error) { t.Logf("Open()") - cfg := config.(rclone.Config) return rclone.Open(cfg, nil) }, } diff --git a/internal/backend/rclone/config.go b/internal/backend/rclone/config.go index f8dc0d84d..2071d84e2 100644 --- a/internal/backend/rclone/config.go +++ b/internal/backend/rclone/config.go @@ -34,7 +34,7 @@ func NewConfig() Config { } // ParseConfig parses the string s and extracts the remote server URL. -func ParseConfig(s string) (interface{}, error) { +func ParseConfig(s string) (*Config, error) { if !strings.HasPrefix(s, "rclone:") { return nil, errors.New("invalid rclone backend specification") } @@ -42,5 +42,5 @@ func ParseConfig(s string) (interface{}, error) { s = s[7:] cfg := NewConfig() cfg.Remote = s - return cfg, nil + return &cfg, nil } diff --git a/internal/backend/rclone/config_test.go b/internal/backend/rclone/config_test.go index 923555136..67b983f66 100644 --- a/internal/backend/rclone/config_test.go +++ b/internal/backend/rclone/config_test.go @@ -1,37 +1,24 @@ package rclone import ( - "reflect" "testing" + + "github.com/restic/restic/internal/backend/test" ) -func TestParseConfig(t *testing.T) { - var tests = []struct { - s string - cfg Config - }{ - { - "rclone:local:foo:/bar", - Config{ - Remote: "local:foo:/bar", - Program: defaultConfig.Program, - Args: defaultConfig.Args, - Connections: defaultConfig.Connections, - Timeout: defaultConfig.Timeout, - }, +var configTests = []test.ConfigTestData[Config]{ + { + S: "rclone:local:foo:/bar", + Cfg: Config{ + Remote: "local:foo:/bar", + Program: defaultConfig.Program, + Args: defaultConfig.Args, + Connections: defaultConfig.Connections, + Timeout: defaultConfig.Timeout, }, - } - - for _, test := range tests { - t.Run("", func(t *testing.T) { - cfg, err := ParseConfig(test.s) - if err != nil { - t.Fatal(err) - } - - if !reflect.DeepEqual(cfg, test.cfg) { - t.Fatalf("wrong config, want:\n %v\ngot:\n %v", test.cfg, cfg) - } - }) - } + }, +} + +func TestParseConfig(t *testing.T) { + test.ParseConfigTester(t, ParseConfig, configTests) } diff --git a/internal/backend/rest/config.go b/internal/backend/rest/config.go index 388153fce..ba42a0220 100644 --- a/internal/backend/rest/config.go +++ b/internal/backend/rest/config.go @@ -26,7 +26,7 @@ func NewConfig() Config { } // ParseConfig parses the string s and extracts the REST server URL. -func ParseConfig(s string) (interface{}, error) { +func ParseConfig(s string) (*Config, error) { if !strings.HasPrefix(s, "rest:") { return nil, errors.New("invalid REST backend specification") } @@ -40,7 +40,7 @@ func ParseConfig(s string) (interface{}, error) { cfg := NewConfig() cfg.URL = u - return cfg, nil + return &cfg, nil } // StripPassword removes the password from the URL diff --git a/internal/backend/rest/config_test.go b/internal/backend/rest/config_test.go index 2d8e32a73..8cfc78407 100644 --- a/internal/backend/rest/config_test.go +++ b/internal/backend/rest/config_test.go @@ -2,8 +2,9 @@ package rest import ( "net/url" - "reflect" "testing" + + "github.com/restic/restic/internal/backend/test" ) func parseURL(s string) *url.URL { @@ -15,20 +16,17 @@ func parseURL(s string) *url.URL { return u } -var configTests = []struct { - s string - cfg Config -}{ +var configTests = []test.ConfigTestData[Config]{ { - s: "rest:http://localhost:1234", - cfg: Config{ + S: "rest:http://localhost:1234", + Cfg: Config{ URL: parseURL("http://localhost:1234/"), Connections: 5, }, }, { - s: "rest:http://localhost:1234/", - cfg: Config{ + S: "rest:http://localhost:1234/", + Cfg: Config{ URL: parseURL("http://localhost:1234/"), Connections: 5, }, @@ -36,17 +34,5 @@ var configTests = []struct { } func TestParseConfig(t *testing.T) { - for _, test := range configTests { - t.Run("", func(t *testing.T) { - cfg, err := ParseConfig(test.s) - if err != nil { - t.Fatalf("%s failed: %v", test.s, err) - } - - if !reflect.DeepEqual(cfg, test.cfg) { - t.Fatalf("\ninput: %s\n wrong config, want:\n %v\ngot:\n %v", - test.s, test.cfg, cfg) - } - }) - } + test.ParseConfigTester(t, ParseConfig, configTests) } diff --git a/internal/backend/rest/rest_test.go b/internal/backend/rest/rest_test.go index 9d7073bc5..2ebd00f5e 100644 --- a/internal/backend/rest/rest_test.go +++ b/internal/backend/rest/rest_test.go @@ -67,36 +67,34 @@ func runRESTServer(ctx context.Context, t testing.TB, dir string) (*url.URL, fun return url, cleanup } -func newTestSuite(_ context.Context, t testing.TB, url *url.URL, minimalData bool) *test.Suite { +func newTestSuite(_ context.Context, t testing.TB, url *url.URL, minimalData bool) *test.Suite[rest.Config] { tr, err := backend.Transport(backend.TransportOptions{}) if err != nil { t.Fatalf("cannot create transport for tests: %v", err) } - return &test.Suite{ + return &test.Suite[rest.Config]{ MinimalData: minimalData, // NewConfig returns a config for a new temporary backend that will be used in tests. - NewConfig: func() (interface{}, error) { + NewConfig: func() (*rest.Config, error) { cfg := rest.NewConfig() cfg.URL = url - return cfg, nil + return &cfg, nil }, // CreateFn is a function that creates a temporary repository for the tests. - Create: func(config interface{}) (restic.Backend, error) { - cfg := config.(rest.Config) + Create: func(cfg rest.Config) (restic.Backend, error) { return rest.Create(context.TODO(), cfg, tr) }, // OpenFn is a function that opens a previously created temporary repository. - Open: func(config interface{}) (restic.Backend, error) { - cfg := config.(rest.Config) + Open: func(cfg rest.Config) (restic.Backend, error) { return rest.Open(cfg, tr) }, // CleanupFn removes data created during the tests. - Cleanup: func(config interface{}) error { + Cleanup: func(cfg rest.Config) error { return nil }, } @@ -130,12 +128,10 @@ func TestBackendRESTExternalServer(t *testing.T) { t.Fatal(err) } - c := cfg.(rest.Config) - ctx, cancel := context.WithCancel(context.Background()) defer cancel() - newTestSuite(ctx, t, c.URL, true).RunTests(t) + newTestSuite(ctx, t, cfg.URL, true).RunTests(t) } func BenchmarkBackendREST(t *testing.B) { diff --git a/internal/backend/s3/config.go b/internal/backend/s3/config.go index 9050e20f4..525373d16 100644 --- a/internal/backend/s3/config.go +++ b/internal/backend/s3/config.go @@ -2,11 +2,13 @@ package s3 import ( "net/url" + "os" "path" "strings" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" + "github.com/restic/restic/internal/restic" ) // Config contains all configuration necessary to connect to an s3 compatible @@ -44,7 +46,7 @@ func init() { // supported configuration formats are s3://host/bucketname/prefix and // s3:host/bucketname/prefix. The host can also be a valid s3 region // name. If no prefix is given the prefix "restic" will be used. -func ParseConfig(s string) (interface{}, error) { +func ParseConfig(s string) (*Config, error) { switch { case strings.HasPrefix(s, "s3:http"): // assume that a URL has been specified, parse it and @@ -75,7 +77,7 @@ func ParseConfig(s string) (interface{}, error) { return createConfig(endpoint, bucket, prefix, false) } -func createConfig(endpoint, bucket, prefix string, useHTTP bool) (interface{}, error) { +func createConfig(endpoint, bucket, prefix string, useHTTP bool) (*Config, error) { if endpoint == "" { return nil, errors.New("s3: invalid format, host/region or bucket name not found") } @@ -89,5 +91,30 @@ func createConfig(endpoint, bucket, prefix string, useHTTP bool) (interface{}, e cfg.UseHTTP = useHTTP cfg.Bucket = bucket cfg.Prefix = prefix - return cfg, nil + return &cfg, nil +} + +var _ restic.ApplyEnvironmenter = &Config{} + +// ApplyEnvironment saves values from the environment to the config. +func (cfg *Config) ApplyEnvironment(prefix string) error { + if cfg.KeyID == "" { + cfg.KeyID = os.Getenv(prefix + "AWS_ACCESS_KEY_ID") + } + + if cfg.Secret.String() == "" { + cfg.Secret = options.NewSecretString(os.Getenv(prefix + "AWS_SECRET_ACCESS_KEY")) + } + + if cfg.KeyID == "" && cfg.Secret.String() != "" { + return errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty") + } else if cfg.KeyID != "" && cfg.Secret.String() == "" { + return errors.Fatalf("unable to open S3 backend: Secret ($AWS_SECRET_ACCESS_KEY) is empty") + } + + if cfg.Region == "" { + cfg.Region = os.Getenv(prefix + "AWS_DEFAULT_REGION") + } + + return nil } diff --git a/internal/backend/s3/config_test.go b/internal/backend/s3/config_test.go index 821fbc244..21fbb27b9 100644 --- a/internal/backend/s3/config_test.go +++ b/internal/backend/s3/config_test.go @@ -3,94 +3,93 @@ package s3 import ( "strings" "testing" + + "github.com/restic/restic/internal/backend/test" ) -var configTests = []struct { - s string - cfg Config -}{ - {"s3://eu-central-1/bucketname", Config{ +var configTests = []test.ConfigTestData[Config]{ + {S: "s3://eu-central-1/bucketname", Cfg: Config{ Endpoint: "eu-central-1", Bucket: "bucketname", Prefix: "", Connections: 5, }}, - {"s3://eu-central-1/bucketname/", Config{ + {S: "s3://eu-central-1/bucketname/", Cfg: Config{ Endpoint: "eu-central-1", Bucket: "bucketname", Prefix: "", Connections: 5, }}, - {"s3://eu-central-1/bucketname/prefix/directory", Config{ + {S: "s3://eu-central-1/bucketname/prefix/directory", Cfg: Config{ Endpoint: "eu-central-1", Bucket: "bucketname", Prefix: "prefix/directory", Connections: 5, }}, - {"s3://eu-central-1/bucketname/prefix/directory/", Config{ + {S: "s3://eu-central-1/bucketname/prefix/directory/", Cfg: Config{ Endpoint: "eu-central-1", Bucket: "bucketname", Prefix: "prefix/directory", Connections: 5, }}, - {"s3:eu-central-1/foobar", Config{ + {S: "s3:eu-central-1/foobar", Cfg: Config{ Endpoint: "eu-central-1", Bucket: "foobar", Prefix: "", Connections: 5, }}, - {"s3:eu-central-1/foobar/", Config{ + {S: "s3:eu-central-1/foobar/", Cfg: Config{ Endpoint: "eu-central-1", Bucket: "foobar", Prefix: "", Connections: 5, }}, - {"s3:eu-central-1/foobar/prefix/directory", Config{ + {S: "s3:eu-central-1/foobar/prefix/directory", Cfg: Config{ Endpoint: "eu-central-1", Bucket: "foobar", Prefix: "prefix/directory", Connections: 5, }}, - {"s3:eu-central-1/foobar/prefix/directory/", Config{ + {S: "s3:eu-central-1/foobar/prefix/directory/", Cfg: Config{ Endpoint: "eu-central-1", Bucket: "foobar", Prefix: "prefix/directory", Connections: 5, }}, - {"s3:https://hostname:9999/foobar", Config{ + {S: "s3:https://hostname:9999/foobar", Cfg: Config{ Endpoint: "hostname:9999", Bucket: "foobar", Prefix: "", Connections: 5, }}, - {"s3:https://hostname:9999/foobar/", Config{ + {S: "s3:https://hostname:9999/foobar/", Cfg: Config{ Endpoint: "hostname:9999", Bucket: "foobar", Prefix: "", Connections: 5, }}, - {"s3:http://hostname:9999/foobar", Config{ + {S: "s3:http://hostname:9999/foobar", Cfg: Config{ Endpoint: "hostname:9999", Bucket: "foobar", Prefix: "", UseHTTP: true, Connections: 5, }}, - {"s3:http://hostname:9999/foobar/", Config{ + {S: "s3:http://hostname:9999/foobar/", Cfg: Config{ Endpoint: "hostname:9999", Bucket: "foobar", Prefix: "", UseHTTP: true, Connections: 5, }}, - {"s3:http://hostname:9999/bucket/prefix/directory", Config{ + {S: "s3:http://hostname:9999/bucket/prefix/directory", Cfg: Config{ Endpoint: "hostname:9999", Bucket: "bucket", Prefix: "prefix/directory", UseHTTP: true, Connections: 5, }}, - {"s3:http://hostname:9999/bucket/prefix/directory/", Config{ + {S: "s3:http://hostname:9999/bucket/prefix/directory/", Cfg: Config{ Endpoint: "hostname:9999", Bucket: "bucket", Prefix: "prefix/directory", @@ -100,19 +99,7 @@ var configTests = []struct { } func TestParseConfig(t *testing.T) { - for i, test := range configTests { - cfg, err := ParseConfig(test.s) - if err != nil { - t.Errorf("test %d:%s failed: %v", i, test.s, err) - continue - } - - if cfg != test.cfg { - t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v", - i, test.s, test.cfg, cfg) - continue - } - } + test.ParseConfigTester(t, ParseConfig, configTests) } func TestParseError(t *testing.T) { diff --git a/internal/backend/s3/s3_test.go b/internal/backend/s3/s3_test.go index c024251a9..1cdc6d7e9 100644 --- a/internal/backend/s3/s3_test.go +++ b/internal/backend/s3/s3_test.go @@ -120,15 +120,15 @@ func createS3(t testing.TB, cfg MinioTestConfig, tr http.RoundTripper) (be resti return be, err } -func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite { +func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite[MinioTestConfig] { tr, err := backend.Transport(backend.TransportOptions{}) if err != nil { t.Fatalf("cannot create transport for tests: %v", err) } - return &test.Suite{ + return &test.Suite[MinioTestConfig]{ // NewConfig returns a config for a new temporary backend that will be used in tests. - NewConfig: func() (interface{}, error) { + NewConfig: func() (*MinioTestConfig, error) { cfg := MinioTestConfig{} cfg.tempdir = rtest.TempDir(t) @@ -142,13 +142,11 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite { cfg.Config.UseHTTP = true cfg.Config.KeyID = key cfg.Config.Secret = options.NewSecretString(secret) - return cfg, nil + return &cfg, nil }, // CreateFn is a function that creates a temporary repository for the tests. - Create: func(config interface{}) (restic.Backend, error) { - cfg := config.(MinioTestConfig) - + Create: func(cfg MinioTestConfig) (restic.Backend, error) { be, err := createS3(t, cfg, tr) if err != nil { return nil, err @@ -167,14 +165,12 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite { }, // OpenFn is a function that opens a previously created temporary repository. - Open: func(config interface{}) (restic.Backend, error) { - cfg := config.(MinioTestConfig) + Open: func(cfg MinioTestConfig) (restic.Backend, error) { return s3.Open(ctx, cfg.Config, tr) }, // CleanupFn removes data created during the tests. - Cleanup: func(config interface{}) error { - cfg := config.(MinioTestConfig) + Cleanup: func(cfg MinioTestConfig) error { if cfg.stopServer != nil { cfg.stopServer() } @@ -217,24 +213,23 @@ func BenchmarkBackendMinio(t *testing.B) { newMinioTestSuite(ctx, t).RunBenchmarks(t) } -func newS3TestSuite(t testing.TB) *test.Suite { +func newS3TestSuite(t testing.TB) *test.Suite[s3.Config] { tr, err := backend.Transport(backend.TransportOptions{}) if err != nil { t.Fatalf("cannot create transport for tests: %v", err) } - return &test.Suite{ + return &test.Suite[s3.Config]{ // do not use excessive data MinimalData: true, // NewConfig returns a config for a new temporary backend that will be used in tests. - NewConfig: func() (interface{}, error) { - s3cfg, err := s3.ParseConfig(os.Getenv("RESTIC_TEST_S3_REPOSITORY")) + NewConfig: func() (*s3.Config, error) { + cfg, err := s3.ParseConfig(os.Getenv("RESTIC_TEST_S3_REPOSITORY")) if err != nil { return nil, err } - cfg := s3cfg.(s3.Config) cfg.KeyID = os.Getenv("RESTIC_TEST_S3_KEY") cfg.Secret = options.NewSecretString(os.Getenv("RESTIC_TEST_S3_SECRET")) cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) @@ -242,9 +237,7 @@ func newS3TestSuite(t testing.TB) *test.Suite { }, // CreateFn is a function that creates a temporary repository for the tests. - Create: func(config interface{}) (restic.Backend, error) { - cfg := config.(s3.Config) - + Create: func(cfg s3.Config) (restic.Backend, error) { be, err := s3.Create(context.TODO(), cfg, tr) if err != nil { return nil, err @@ -263,15 +256,12 @@ func newS3TestSuite(t testing.TB) *test.Suite { }, // OpenFn is a function that opens a previously created temporary repository. - Open: func(config interface{}) (restic.Backend, error) { - cfg := config.(s3.Config) + Open: func(cfg s3.Config) (restic.Backend, error) { return s3.Open(context.TODO(), cfg, tr) }, // CleanupFn removes data created during the tests. - Cleanup: func(config interface{}) error { - cfg := config.(s3.Config) - + Cleanup: func(cfg s3.Config) error { be, err := s3.Open(context.TODO(), cfg, tr) if err != nil { return err diff --git a/internal/backend/sftp/config.go b/internal/backend/sftp/config.go index 1a7309de3..ed7c2cafa 100644 --- a/internal/backend/sftp/config.go +++ b/internal/backend/sftp/config.go @@ -35,7 +35,7 @@ func init() { // and sftp:user@host:directory. The directory will be path Cleaned and can // be an absolute path if it starts with a '/' (e.g. // sftp://user@host//absolute and sftp:user@host:/absolute). -func ParseConfig(s string) (interface{}, error) { +func ParseConfig(s string) (*Config, error) { var user, host, port, dir string switch { case strings.HasPrefix(s, "sftp://"): @@ -89,5 +89,5 @@ func ParseConfig(s string) (interface{}, error) { cfg.Port = port cfg.Path = p - return cfg, nil + return &cfg, nil } diff --git a/internal/backend/sftp/config_test.go b/internal/backend/sftp/config_test.go index 3772c038b..bf7fa9653 100644 --- a/internal/backend/sftp/config_test.go +++ b/internal/backend/sftp/config_test.go @@ -2,94 +2,81 @@ package sftp import ( "testing" + + "github.com/restic/restic/internal/backend/test" ) -var configTests = []struct { - in string - cfg Config -}{ +var configTests = []test.ConfigTestData[Config]{ // first form, user specified sftp://user@host/dir { - "sftp://user@host/dir/subdir", - Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5}, + S: "sftp://user@host/dir/subdir", + Cfg: Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5}, }, { - "sftp://host/dir/subdir", - Config{Host: "host", Path: "dir/subdir", Connections: 5}, + S: "sftp://host/dir/subdir", + Cfg: Config{Host: "host", Path: "dir/subdir", Connections: 5}, }, { - "sftp://host//dir/subdir", - Config{Host: "host", Path: "/dir/subdir", Connections: 5}, + S: "sftp://host//dir/subdir", + Cfg: Config{Host: "host", Path: "/dir/subdir", Connections: 5}, }, { - "sftp://host:10022//dir/subdir", - Config{Host: "host", Port: "10022", Path: "/dir/subdir", Connections: 5}, + S: "sftp://host:10022//dir/subdir", + Cfg: Config{Host: "host", Port: "10022", Path: "/dir/subdir", Connections: 5}, }, { - "sftp://user@host:10022//dir/subdir", - Config{User: "user", Host: "host", Port: "10022", Path: "/dir/subdir", Connections: 5}, + S: "sftp://user@host:10022//dir/subdir", + Cfg: Config{User: "user", Host: "host", Port: "10022", Path: "/dir/subdir", Connections: 5}, }, { - "sftp://user@host/dir/subdir/../other", - Config{User: "user", Host: "host", Path: "dir/other", Connections: 5}, + S: "sftp://user@host/dir/subdir/../other", + Cfg: Config{User: "user", Host: "host", Path: "dir/other", Connections: 5}, }, { - "sftp://user@host/dir///subdir", - Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5}, + S: "sftp://user@host/dir///subdir", + Cfg: Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5}, }, // IPv6 address. { - "sftp://user@[::1]/dir", - Config{User: "user", Host: "::1", Path: "dir", Connections: 5}, + S: "sftp://user@[::1]/dir", + Cfg: Config{User: "user", Host: "::1", Path: "dir", Connections: 5}, }, // IPv6 address with port. { - "sftp://user@[::1]:22/dir", - Config{User: "user", Host: "::1", Port: "22", Path: "dir", Connections: 5}, + S: "sftp://user@[::1]:22/dir", + Cfg: Config{User: "user", Host: "::1", Port: "22", Path: "dir", Connections: 5}, }, // second form, user specified sftp:user@host:/dir { - "sftp:user@host:/dir/subdir", - Config{User: "user", Host: "host", Path: "/dir/subdir", Connections: 5}, + S: "sftp:user@host:/dir/subdir", + Cfg: Config{User: "user", Host: "host", Path: "/dir/subdir", Connections: 5}, }, { - "sftp:user@domain@host:/dir/subdir", - Config{User: "user@domain", Host: "host", Path: "/dir/subdir", Connections: 5}, + S: "sftp:user@domain@host:/dir/subdir", + Cfg: Config{User: "user@domain", Host: "host", Path: "/dir/subdir", Connections: 5}, }, { - "sftp:host:../dir/subdir", - Config{Host: "host", Path: "../dir/subdir", Connections: 5}, + S: "sftp:host:../dir/subdir", + Cfg: Config{Host: "host", Path: "../dir/subdir", Connections: 5}, }, { - "sftp:user@host:dir/subdir:suffix", - Config{User: "user", Host: "host", Path: "dir/subdir:suffix", Connections: 5}, + S: "sftp:user@host:dir/subdir:suffix", + Cfg: Config{User: "user", Host: "host", Path: "dir/subdir:suffix", Connections: 5}, }, { - "sftp:user@host:dir/subdir/../other", - Config{User: "user", Host: "host", Path: "dir/other", Connections: 5}, + S: "sftp:user@host:dir/subdir/../other", + Cfg: Config{User: "user", Host: "host", Path: "dir/other", Connections: 5}, }, { - "sftp:user@host:dir///subdir", - Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5}, + S: "sftp:user@host:dir///subdir", + Cfg: Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5}, }, } func TestParseConfig(t *testing.T) { - for i, test := range configTests { - cfg, err := ParseConfig(test.in) - if err != nil { - t.Errorf("test %d:%s failed: %v", i, test.in, err) - continue - } - - if cfg != test.cfg { - t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v", - i, test.in, test.cfg, cfg) - continue - } - } + test.ParseConfigTester(t, ParseConfig, configTests) } var configTestsInvalid = []string{ diff --git a/internal/backend/sftp/sftp_test.go b/internal/backend/sftp/sftp_test.go index 0dbcd291c..98175ca26 100644 --- a/internal/backend/sftp/sftp_test.go +++ b/internal/backend/sftp/sftp_test.go @@ -29,10 +29,10 @@ func findSFTPServerBinary() string { var sftpServer = findSFTPServerBinary() -func newTestSuite(t testing.TB) *test.Suite { - return &test.Suite{ +func newTestSuite(t testing.TB) *test.Suite[sftp.Config] { + return &test.Suite[sftp.Config]{ // NewConfig returns a config for a new temporary backend that will be used in tests. - NewConfig: func() (interface{}, error) { + NewConfig: func() (*sftp.Config, error) { dir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-sftp-") if err != nil { t.Fatal(err) @@ -40,7 +40,7 @@ func newTestSuite(t testing.TB) *test.Suite { t.Logf("create new backend at %v", dir) - cfg := sftp.Config{ + cfg := &sftp.Config{ Path: dir, Command: fmt.Sprintf("%q -e", sftpServer), Connections: 5, @@ -49,20 +49,17 @@ func newTestSuite(t testing.TB) *test.Suite { }, // CreateFn is a function that creates a temporary repository for the tests. - Create: func(config interface{}) (restic.Backend, error) { - cfg := config.(sftp.Config) + Create: func(cfg sftp.Config) (restic.Backend, error) { return sftp.Create(context.TODO(), cfg) }, // OpenFn is a function that opens a previously created temporary repository. - Open: func(config interface{}) (restic.Backend, error) { - cfg := config.(sftp.Config) + Open: func(cfg sftp.Config) (restic.Backend, error) { return sftp.Open(context.TODO(), cfg) }, // CleanupFn removes data created during the tests. - Cleanup: func(config interface{}) error { - cfg := config.(sftp.Config) + Cleanup: func(cfg sftp.Config) error { if !rtest.TestCleanupTempDirs { t.Logf("leaving test backend dir at %v", cfg.Path) } diff --git a/internal/backend/smb/config.go b/internal/backend/smb/config.go index d6f368b98..1d9f9d7ac 100644 --- a/internal/backend/smb/config.go +++ b/internal/backend/smb/config.go @@ -2,6 +2,7 @@ package smb import ( "net/url" + "os" "path" "strconv" "strings" @@ -9,6 +10,7 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" + "github.com/restic/restic/internal/restic" ) // Config contains all configuration necessary to connect to an SMB server @@ -54,7 +56,7 @@ func init() { // ParseConfig parses the string s and extracts the s3 config. The // supported configuration format is smb://[user@]host[:port]/sharename/directory. // User and port are optional. Default port is 445. -func ParseConfig(s string) (interface{}, error) { +func ParseConfig(s string) (*Config, error) { var repo string switch { case strings.HasPrefix(s, "smb://"): @@ -106,7 +108,7 @@ func ParseConfig(s string) (interface{}, error) { return createConfig(user, host, portNum, sharename, directory) } -func createConfig(user string, host string, port int, sharename, directory string) (interface{}, error) { +func createConfig(user string, host string, port int, sharename, directory string) (*Config, error) { if host == "" { return nil, errors.New("smb: invalid format, Host not found") } @@ -121,5 +123,26 @@ func createConfig(user string, host string, port int, sharename, directory strin cfg.Port = port cfg.ShareName = sharename cfg.Path = directory - return cfg, nil + return &cfg, nil +} + +var _ restic.ApplyEnvironmenter = &Config{} + +// ApplyEnvironment saves values from the environment to the config. +func (cfg *Config) ApplyEnvironment(prefix string) error { + if cfg.User == "" { + cfg.User = os.Getenv("RESTIC_SMB_USER") + } + + if cfg.Password.String() == "" { + cfg.Password = options.NewSecretString(os.Getenv("RESTIC_SMB_PASSWORD")) + } + + if cfg.Domain == "" { + cfg.Domain = os.Getenv("RESTIC_SMB_DOMAIN") + } + if cfg.Domain == "" { + cfg.Domain = DefaultDomain + } + return nil } diff --git a/internal/backend/smb/config_test.go b/internal/backend/smb/config_test.go index 32f02c7bd..9b3e1dee2 100644 --- a/internal/backend/smb/config_test.go +++ b/internal/backend/smb/config_test.go @@ -3,76 +3,68 @@ package smb import ( "strings" "testing" + + "github.com/restic/restic/internal/backend/test" ) -var configTests = []struct { - s string - cfg Config -}{ - {"smb://user@host/sharename/directory", Config{ - Host: "host", - Port: DefaultSmbPort, - User: "user", - Domain: DefaultDomain, - ShareName: "sharename", - Path: "directory", - Connections: DefaultConnections, - IdleTimeout: DefaultIdleTimeout, - }}, - {"smb://user@host:456/sharename/directory", Config{ - Host: "host", - Port: 456, - User: "user", - Domain: DefaultDomain, - ShareName: "sharename", - Path: "directory", - Connections: DefaultConnections, - IdleTimeout: DefaultIdleTimeout, - }}, - {"smb://host/sharename/directory", Config{ - Host: "host", - Port: DefaultSmbPort, - Domain: DefaultDomain, - ShareName: "sharename", - Path: "directory", - Connections: DefaultConnections, - IdleTimeout: DefaultIdleTimeout, - }}, - {"smb://host:446/sharename/directory", Config{ - Host: "host", - Port: 446, - Domain: DefaultDomain, - ShareName: "sharename", - Path: "directory", - Connections: DefaultConnections, - IdleTimeout: DefaultIdleTimeout, - }}, - {"smb:user@host:466/sharename/directory", Config{ - Host: "host", - Port: 466, - User: "user", - Domain: DefaultDomain, - ShareName: "sharename", - Path: "directory", - Connections: DefaultConnections, - IdleTimeout: DefaultIdleTimeout, - }}, +var configTests = []test.ConfigTestData[Config]{ + {S: "smb://user@host/sharename/directory", + Cfg: Config{ + Host: "host", + Port: DefaultSmbPort, + User: "user", + Domain: DefaultDomain, + ShareName: "sharename", + Path: "directory", + Connections: DefaultConnections, + IdleTimeout: DefaultIdleTimeout, + }}, + {S: "smb://user@host:456/sharename/directory", + Cfg: Config{ + Host: "host", + Port: 456, + User: "user", + Domain: DefaultDomain, + ShareName: "sharename", + Path: "directory", + Connections: DefaultConnections, + IdleTimeout: DefaultIdleTimeout, + }}, + {S: "smb://host/sharename/directory", + Cfg: Config{ + Host: "host", + Port: DefaultSmbPort, + Domain: DefaultDomain, + ShareName: "sharename", + Path: "directory", + Connections: DefaultConnections, + IdleTimeout: DefaultIdleTimeout, + }}, + {S: "smb://host:446/sharename/directory", + Cfg: Config{ + Host: "host", + Port: 446, + Domain: DefaultDomain, + ShareName: "sharename", + Path: "directory", + Connections: DefaultConnections, + IdleTimeout: DefaultIdleTimeout, + }}, + {S: "smb:user@host:466/sharename/directory", + Cfg: Config{ + Host: "host", + Port: 466, + User: "user", + Domain: DefaultDomain, + ShareName: "sharename", + Path: "directory", + Connections: DefaultConnections, + IdleTimeout: DefaultIdleTimeout, + }}, } func TestParseConfig(t *testing.T) { - for i, test := range configTests { - cfg, err := ParseConfig(test.s) - if err != nil { - t.Errorf("test %d:%s failed: %v", i, test.s, err) - continue - } - - if cfg != test.cfg { - t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v", - i, test.s, test.cfg, cfg) - continue - } - } + test.ParseConfigTester(t, ParseConfig, configTests) } func TestParseError(t *testing.T) { diff --git a/internal/backend/smb/smb_test.go b/internal/backend/smb/smb_test.go index c210bb51b..2b81a5fbb 100644 --- a/internal/backend/smb/smb_test.go +++ b/internal/backend/smb/smb_test.go @@ -13,12 +13,12 @@ import ( rtest "github.com/restic/restic/internal/test" ) -func newTestSuite(t testing.TB) *test.Suite { - return &test.Suite{ +func newTestSuite(t testing.TB) *test.Suite[smb.Config] { + return &test.Suite[smb.Config]{ // NewConfig returns a config for a new temporary backend that will be used in tests. - NewConfig: func() (interface{}, error) { + NewConfig: func() (*smb.Config, error) { - cfg := smb.NewConfig() + cfg := &smb.Config{} cfg.Host = "127.0.0.1" cfg.User = "smbuser" cfg.ShareName = cfg.User @@ -38,20 +38,17 @@ func newTestSuite(t testing.TB) *test.Suite { }, // CreateFn is a function that creates a temporary repository for the tests. - Create: func(config interface{}) (restic.Backend, error) { - cfg := config.(smb.Config) + Create: func(cfg smb.Config) (restic.Backend, error) { return smb.Create(context.TODO(), cfg) }, // OpenFn is a function that opens a previously created temporary repository. - Open: func(config interface{}) (restic.Backend, error) { - cfg := config.(smb.Config) + Open: func(cfg smb.Config) (restic.Backend, error) { return smb.Open(context.TODO(), cfg) }, // CleanupFn removes data created during the tests. - Cleanup: func(config interface{}) error { - cfg := config.(smb.Config) + Cleanup: func(cfg smb.Config) error { if !rtest.TestCleanupTempDirs { t.Logf("leaving test backend dir at %v", cfg.Path) } diff --git a/internal/backend/swift/config.go b/internal/backend/swift/config.go index ced256752..b9f5d3995 100644 --- a/internal/backend/swift/config.go +++ b/internal/backend/swift/config.go @@ -6,6 +6,7 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" + "github.com/restic/restic/internal/restic" ) // Config contains basic configuration needed to specify swift location for a swift server @@ -50,7 +51,7 @@ func NewConfig() Config { } // ParseConfig parses the string s and extract swift's container name and prefix. -func ParseConfig(s string) (interface{}, error) { +func ParseConfig(s string) (*Config, error) { if !strings.HasPrefix(s, "swift:") { return nil, errors.New("invalid URL, expected: swift:container-name:/[prefix]") } @@ -70,48 +71,49 @@ func ParseConfig(s string) (interface{}, error) { cfg.Container = container cfg.Prefix = prefix - return cfg, nil + return &cfg, nil } +var _ restic.ApplyEnvironmenter = &Config{} + // ApplyEnvironment saves values from the environment to the config. -func ApplyEnvironment(prefix string, cfg interface{}) error { - c := cfg.(*Config) +func (cfg *Config) ApplyEnvironment(prefix string) error { for _, val := range []struct { s *string env string }{ // v2/v3 specific - {&c.UserName, prefix + "OS_USERNAME"}, - {&c.APIKey, prefix + "OS_PASSWORD"}, - {&c.Region, prefix + "OS_REGION_NAME"}, - {&c.AuthURL, prefix + "OS_AUTH_URL"}, + {&cfg.UserName, prefix + "OS_USERNAME"}, + {&cfg.APIKey, prefix + "OS_PASSWORD"}, + {&cfg.Region, prefix + "OS_REGION_NAME"}, + {&cfg.AuthURL, prefix + "OS_AUTH_URL"}, // v3 specific - {&c.UserID, prefix + "OS_USER_ID"}, - {&c.Domain, prefix + "OS_USER_DOMAIN_NAME"}, - {&c.DomainID, prefix + "OS_USER_DOMAIN_ID"}, - {&c.Tenant, prefix + "OS_PROJECT_NAME"}, - {&c.TenantDomain, prefix + "OS_PROJECT_DOMAIN_NAME"}, - {&c.TenantDomainID, prefix + "OS_PROJECT_DOMAIN_ID"}, - {&c.TrustID, prefix + "OS_TRUST_ID"}, + {&cfg.UserID, prefix + "OS_USER_ID"}, + {&cfg.Domain, prefix + "OS_USER_DOMAIN_NAME"}, + {&cfg.DomainID, prefix + "OS_USER_DOMAIN_ID"}, + {&cfg.Tenant, prefix + "OS_PROJECT_NAME"}, + {&cfg.TenantDomain, prefix + "OS_PROJECT_DOMAIN_NAME"}, + {&cfg.TenantDomainID, prefix + "OS_PROJECT_DOMAIN_ID"}, + {&cfg.TrustID, prefix + "OS_TRUST_ID"}, // v2 specific - {&c.TenantID, prefix + "OS_TENANT_ID"}, - {&c.Tenant, prefix + "OS_TENANT_NAME"}, + {&cfg.TenantID, prefix + "OS_TENANT_ID"}, + {&cfg.Tenant, prefix + "OS_TENANT_NAME"}, // v1 specific - {&c.AuthURL, prefix + "ST_AUTH"}, - {&c.UserName, prefix + "ST_USER"}, - {&c.APIKey, prefix + "ST_KEY"}, + {&cfg.AuthURL, prefix + "ST_AUTH"}, + {&cfg.UserName, prefix + "ST_USER"}, + {&cfg.APIKey, prefix + "ST_KEY"}, // Application Credential auth - {&c.ApplicationCredentialID, prefix + "OS_APPLICATION_CREDENTIAL_ID"}, - {&c.ApplicationCredentialName, prefix + "OS_APPLICATION_CREDENTIAL_NAME"}, + {&cfg.ApplicationCredentialID, prefix + "OS_APPLICATION_CREDENTIAL_ID"}, + {&cfg.ApplicationCredentialName, prefix + "OS_APPLICATION_CREDENTIAL_NAME"}, // Manual authentication - {&c.StorageURL, prefix + "OS_STORAGE_URL"}, + {&cfg.StorageURL, prefix + "OS_STORAGE_URL"}, - {&c.DefaultContainerPolicy, prefix + "SWIFT_DEFAULT_CONTAINER_POLICY"}, + {&cfg.DefaultContainerPolicy, prefix + "SWIFT_DEFAULT_CONTAINER_POLICY"}, } { if *val.s == "" { *val.s = os.Getenv(val.env) @@ -121,8 +123,8 @@ func ApplyEnvironment(prefix string, cfg interface{}) error { s *options.SecretString env string }{ - {&c.ApplicationCredentialSecret, prefix + "OS_APPLICATION_CREDENTIAL_SECRET"}, - {&c.AuthToken, prefix + "OS_AUTH_TOKEN"}, + {&cfg.ApplicationCredentialSecret, prefix + "OS_APPLICATION_CREDENTIAL_SECRET"}, + {&cfg.AuthToken, prefix + "OS_AUTH_TOKEN"}, } { if val.s.String() == "" { *val.s = options.NewSecretString(os.Getenv(val.env)) diff --git a/internal/backend/swift/config_test.go b/internal/backend/swift/config_test.go index 35f091a9b..3e094f9ea 100644 --- a/internal/backend/swift/config_test.go +++ b/internal/backend/swift/config_test.go @@ -1,29 +1,30 @@ package swift -import "testing" +import ( + "testing" -var configTests = []struct { - s string - cfg Config -}{ + "github.com/restic/restic/internal/backend/test" +) + +var configTests = []test.ConfigTestData[Config]{ { - "swift:cnt1:/", - Config{ + S: "swift:cnt1:/", + Cfg: Config{ Container: "cnt1", Prefix: "", Connections: 5, }, }, { - "swift:cnt2:/prefix", - Config{Container: "cnt2", + S: "swift:cnt2:/prefix", + Cfg: Config{Container: "cnt2", Prefix: "prefix", Connections: 5, }, }, { - "swift:cnt3:/prefix/longer", - Config{Container: "cnt3", + S: "swift:cnt3:/prefix/longer", + Cfg: Config{Container: "cnt3", Prefix: "prefix/longer", Connections: 5, }, @@ -31,24 +32,7 @@ var configTests = []struct { } func TestParseConfig(t *testing.T) { - for _, test := range configTests { - t.Run("", func(t *testing.T) { - v, err := ParseConfig(test.s) - if err != nil { - t.Fatalf("parsing %q failed: %v", test.s, err) - } - - cfg, ok := v.(Config) - if !ok { - t.Fatalf("wrong type returned, want Config, got %T", cfg) - } - - if cfg != test.cfg { - t.Fatalf("wrong output for %q, want:\n %#v\ngot:\n %#v", - test.s, test.cfg, cfg) - } - }) - } + test.ParseConfigTester(t, ParseConfig, configTests) } var configTestsInvalid = []string{ diff --git a/internal/backend/swift/swift_test.go b/internal/backend/swift/swift_test.go index 0912e4f7e..cb0992010 100644 --- a/internal/backend/swift/swift_test.go +++ b/internal/backend/swift/swift_test.go @@ -15,13 +15,13 @@ import ( rtest "github.com/restic/restic/internal/test" ) -func newSwiftTestSuite(t testing.TB) *test.Suite { +func newSwiftTestSuite(t testing.TB) *test.Suite[swift.Config] { tr, err := backend.Transport(backend.TransportOptions{}) if err != nil { t.Fatalf("cannot create transport for tests: %v", err) } - return &test.Suite{ + return &test.Suite[swift.Config]{ // do not use excessive data MinimalData: true, @@ -42,14 +42,13 @@ func newSwiftTestSuite(t testing.TB) *test.Suite { }, // NewConfig returns a config for a new temporary backend that will be used in tests. - NewConfig: func() (interface{}, error) { - swiftcfg, err := swift.ParseConfig(os.Getenv("RESTIC_TEST_SWIFT")) + NewConfig: func() (*swift.Config, error) { + cfg, err := swift.ParseConfig(os.Getenv("RESTIC_TEST_SWIFT")) if err != nil { return nil, err } - cfg := swiftcfg.(swift.Config) - if err = swift.ApplyEnvironment("RESTIC_TEST_", &cfg); err != nil { + if err = cfg.ApplyEnvironment("RESTIC_TEST_"); err != nil { return nil, err } cfg.Prefix += fmt.Sprintf("/test-%d", time.Now().UnixNano()) @@ -58,9 +57,7 @@ func newSwiftTestSuite(t testing.TB) *test.Suite { }, // CreateFn is a function that creates a temporary repository for the tests. - Create: func(config interface{}) (restic.Backend, error) { - cfg := config.(swift.Config) - + Create: func(cfg swift.Config) (restic.Backend, error) { be, err := swift.Open(context.TODO(), cfg, tr) if err != nil { return nil, err @@ -79,15 +76,12 @@ func newSwiftTestSuite(t testing.TB) *test.Suite { }, // OpenFn is a function that opens a previously created temporary repository. - Open: func(config interface{}) (restic.Backend, error) { - cfg := config.(swift.Config) + Open: func(cfg swift.Config) (restic.Backend, error) { return swift.Open(context.TODO(), cfg, tr) }, // CleanupFn removes data created during the tests. - Cleanup: func(config interface{}) error { - cfg := config.(swift.Config) - + Cleanup: func(cfg swift.Config) error { be, err := swift.Open(context.TODO(), cfg, tr) if err != nil { return err diff --git a/internal/backend/test/benchmarks.go b/internal/backend/test/benchmarks.go index b977eb682..150ef3987 100644 --- a/internal/backend/test/benchmarks.go +++ b/internal/backend/test/benchmarks.go @@ -29,7 +29,7 @@ func remove(t testing.TB, be restic.Backend, h restic.Handle) { // BenchmarkLoadFile benchmarks the Load() method of a backend by // loading a complete file. -func (s *Suite) BenchmarkLoadFile(t *testing.B) { +func (s *Suite[C]) BenchmarkLoadFile(t *testing.B) { be := s.open(t) defer s.close(t, be) @@ -64,7 +64,7 @@ func (s *Suite) BenchmarkLoadFile(t *testing.B) { // BenchmarkLoadPartialFile benchmarks the Load() method of a backend by // loading the remainder of a file starting at a given offset. -func (s *Suite) BenchmarkLoadPartialFile(t *testing.B) { +func (s *Suite[C]) BenchmarkLoadPartialFile(t *testing.B) { be := s.open(t) defer s.close(t, be) @@ -101,7 +101,7 @@ func (s *Suite) BenchmarkLoadPartialFile(t *testing.B) { // BenchmarkLoadPartialFileOffset benchmarks the Load() method of a // backend by loading a number of bytes of a file starting at a given offset. -func (s *Suite) BenchmarkLoadPartialFileOffset(t *testing.B) { +func (s *Suite[C]) BenchmarkLoadPartialFileOffset(t *testing.B) { be := s.open(t) defer s.close(t, be) @@ -139,7 +139,7 @@ func (s *Suite) BenchmarkLoadPartialFileOffset(t *testing.B) { } // BenchmarkSave benchmarks the Save() method of a backend. -func (s *Suite) BenchmarkSave(t *testing.B) { +func (s *Suite[C]) BenchmarkSave(t *testing.B) { be := s.open(t) defer s.close(t, be) diff --git a/internal/backend/test/config.go b/internal/backend/test/config.go new file mode 100644 index 000000000..496ba2761 --- /dev/null +++ b/internal/backend/test/config.go @@ -0,0 +1,28 @@ +package test + +import ( + "fmt" + "reflect" + "testing" +) + +type ConfigTestData[C comparable] struct { + S string + Cfg C +} + +func ParseConfigTester[C comparable](t *testing.T, parser func(s string) (*C, error), tests []ConfigTestData[C]) { + for i, test := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + cfg, err := parser(test.S) + if err != nil { + t.Fatalf("%s failed: %v", test.S, err) + } + + if !reflect.DeepEqual(*cfg, test.Cfg) { + t.Fatalf("input: %s\n wrong config, want:\n %#v\ngot:\n %#v", + test.S, test.Cfg, *cfg) + } + }) + } +} diff --git a/internal/backend/test/suite.go b/internal/backend/test/suite.go index 45c6d96bd..75ae0630b 100644 --- a/internal/backend/test/suite.go +++ b/internal/backend/test/suite.go @@ -11,21 +11,21 @@ import ( ) // Suite implements a test suite for restic backends. -type Suite struct { +type Suite[C any] struct { // Config should be used to configure the backend. - Config interface{} + Config *C // NewConfig returns a config for a new temporary backend that will be used in tests. - NewConfig func() (interface{}, error) + NewConfig func() (*C, error) // CreateFn is a function that creates a temporary repository for the tests. - Create func(cfg interface{}) (restic.Backend, error) + Create func(cfg C) (restic.Backend, error) // OpenFn is a function that opens a previously created temporary repository. - Open func(cfg interface{}) (restic.Backend, error) + Open func(cfg C) (restic.Backend, error) // CleanupFn removes data created during the tests. - Cleanup func(cfg interface{}) error + Cleanup func(cfg C) error // MinimalData instructs the tests to not use excessive data. MinimalData bool @@ -40,7 +40,7 @@ type Suite struct { } // RunTests executes all defined tests as subtests of t. -func (s *Suite) RunTests(t *testing.T) { +func (s *Suite[C]) RunTests(t *testing.T) { var err error s.Config, err = s.NewConfig() if err != nil { @@ -61,7 +61,7 @@ func (s *Suite) RunTests(t *testing.T) { } if s.Cleanup != nil { - if err = s.Cleanup(s.Config); err != nil { + if err = s.Cleanup(*s.Config); err != nil { t.Fatal(err) } } @@ -72,7 +72,7 @@ type testFunction struct { Fn func(*testing.T) } -func (s *Suite) testFuncs(t testing.TB) (funcs []testFunction) { +func (s *Suite[C]) testFuncs(t testing.TB) (funcs []testFunction) { tpe := reflect.TypeOf(s) v := reflect.ValueOf(s) @@ -107,7 +107,7 @@ type benchmarkFunction struct { Fn func(*testing.B) } -func (s *Suite) benchmarkFuncs(t testing.TB) (funcs []benchmarkFunction) { +func (s *Suite[C]) benchmarkFuncs(t testing.TB) (funcs []benchmarkFunction) { tpe := reflect.TypeOf(s) v := reflect.ValueOf(s) @@ -138,7 +138,7 @@ func (s *Suite) benchmarkFuncs(t testing.TB) (funcs []benchmarkFunction) { } // RunBenchmarks executes all defined benchmarks as subtests of b. -func (s *Suite) RunBenchmarks(b *testing.B) { +func (s *Suite[C]) RunBenchmarks(b *testing.B) { var err error s.Config, err = s.NewConfig() if err != nil { @@ -158,28 +158,28 @@ func (s *Suite) RunBenchmarks(b *testing.B) { return } - if err = s.Cleanup(s.Config); err != nil { + if err = s.Cleanup(*s.Config); err != nil { b.Fatal(err) } } -func (s *Suite) create(t testing.TB) restic.Backend { - be, err := s.Create(s.Config) +func (s *Suite[C]) create(t testing.TB) restic.Backend { + be, err := s.Create(*s.Config) if err != nil { t.Fatal(err) } return be } -func (s *Suite) open(t testing.TB) restic.Backend { - be, err := s.Open(s.Config) +func (s *Suite[C]) open(t testing.TB) restic.Backend { + be, err := s.Open(*s.Config) if err != nil { t.Fatal(err) } return be } -func (s *Suite) close(t testing.TB, be restic.Backend) { +func (s *Suite[C]) close(t testing.TB, be restic.Backend) { err := be.Close() if err != nil { t.Fatal(err) diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go index a9514bf6b..c4462495f 100644 --- a/internal/backend/test/tests.go +++ b/internal/backend/test/tests.go @@ -38,7 +38,7 @@ func beTest(ctx context.Context, be restic.Backend, h restic.Handle) (bool, erro // TestCreateWithConfig tests that creating a backend in a location which already // has a config file fails. -func (s *Suite) TestCreateWithConfig(t *testing.T) { +func (s *Suite[C]) TestCreateWithConfig(t *testing.T) { b := s.open(t) defer s.close(t, b) @@ -57,7 +57,7 @@ func (s *Suite) TestCreateWithConfig(t *testing.T) { store(t, b, restic.ConfigFile, []byte("test config")) // now create the backend again, this must fail - _, err = s.Create(s.Config) + _, err = s.Create(*s.Config) if err == nil { t.Fatalf("expected error not found for creating a backend with an existing config file") } @@ -70,7 +70,7 @@ func (s *Suite) TestCreateWithConfig(t *testing.T) { } // TestLocation tests that a location string is returned. -func (s *Suite) TestLocation(t *testing.T) { +func (s *Suite[C]) TestLocation(t *testing.T) { b := s.open(t) defer s.close(t, b) @@ -81,7 +81,7 @@ func (s *Suite) TestLocation(t *testing.T) { } // TestConfig saves and loads a config from the backend. -func (s *Suite) TestConfig(t *testing.T) { +func (s *Suite[C]) TestConfig(t *testing.T) { b := s.open(t) defer s.close(t, b) @@ -118,7 +118,7 @@ func (s *Suite) TestConfig(t *testing.T) { } // TestLoad tests the backend's Load function. -func (s *Suite) TestLoad(t *testing.T) { +func (s *Suite[C]) TestLoad(t *testing.T) { seedRand(t) b := s.open(t) @@ -222,8 +222,12 @@ func (s *Suite) TestLoad(t *testing.T) { test.OK(t, b.Remove(context.TODO(), handle)) } +type setter interface { + SetListMaxItems(int) +} + // TestList makes sure that the backend implements List() pagination correctly. -func (s *Suite) TestList(t *testing.T) { +func (s *Suite[C]) TestList(t *testing.T) { seedRand(t) numTestFiles := rand.Intn(20) + 20 @@ -269,10 +273,6 @@ func (s *Suite) TestList(t *testing.T) { t.Run(fmt.Sprintf("max-%v", test.maxItems), func(t *testing.T) { list2 := make(map[restic.ID]int64) - type setter interface { - SetListMaxItems(int) - } - if s, ok := b.(setter); ok { t.Logf("setting max list items to %d", test.maxItems) s.SetListMaxItems(test.maxItems) @@ -326,7 +326,7 @@ func (s *Suite) TestList(t *testing.T) { } // TestListCancel tests that the context is respected and the error is returned by List. -func (s *Suite) TestListCancel(t *testing.T) { +func (s *Suite[C]) TestListCancel(t *testing.T) { seedRand(t) numTestFiles := 5 @@ -466,7 +466,7 @@ func (ec errorCloser) Rewind() error { } // TestSave tests saving data in the backend. -func (s *Suite) TestSave(t *testing.T) { +func (s *Suite[C]) TestSave(t *testing.T) { seedRand(t) b := s.open(t) @@ -582,7 +582,7 @@ func (r *incompleteByteReader) Length() int64 { } // TestSaveError tests saving data in the backend. -func (s *Suite) TestSaveError(t *testing.T) { +func (s *Suite[C]) TestSaveError(t *testing.T) { seedRand(t) b := s.open(t) @@ -621,7 +621,7 @@ func (b *wrongByteReader) Hash() []byte { } // TestSaveWrongHash tests that uploads with a wrong hash fail -func (s *Suite) TestSaveWrongHash(t *testing.T) { +func (s *Suite[C]) TestSaveWrongHash(t *testing.T) { seedRand(t) b := s.open(t) @@ -679,7 +679,7 @@ func testLoad(b restic.Backend, h restic.Handle) error { }) } -func (s *Suite) delayedRemove(t testing.TB, be restic.Backend, handles ...restic.Handle) error { +func (s *Suite[C]) delayedRemove(t testing.TB, be restic.Backend, handles ...restic.Handle) error { // Some backend (swift, I'm looking at you) may implement delayed // removal of data. Let's wait a bit if this happens. @@ -746,7 +746,7 @@ func delayedList(t testing.TB, b restic.Backend, tpe restic.FileType, max int, m } // TestBackend tests all functions of the backend. -func (s *Suite) TestBackend(t *testing.T) { +func (s *Suite[C]) TestBackend(t *testing.T) { b := s.open(t) defer s.close(t, b) @@ -867,7 +867,7 @@ func (s *Suite) TestBackend(t *testing.T) { } // TestZZZDelete tests the Delete function. The name ensures that this test is executed last. -func (s *Suite) TestZZZDelete(t *testing.T) { +func (s *Suite[C]) TestZZZDelete(t *testing.T) { if !test.TestCleanupTempDirs { t.Skipf("not removing backend, TestCleanupTempDirs is false") } diff --git a/internal/restorer/preallocate_darwin.go b/internal/fs/preallocate_darwin.go similarity index 87% rename from internal/restorer/preallocate_darwin.go rename to internal/fs/preallocate_darwin.go index ae6e5ee3e..af46e971b 100644 --- a/internal/restorer/preallocate_darwin.go +++ b/internal/fs/preallocate_darwin.go @@ -1,4 +1,4 @@ -package restorer +package fs import ( "os" @@ -6,7 +6,7 @@ import ( "golang.org/x/sys/unix" ) -func preallocateFile(wr *os.File, size int64) error { +func PreallocateFile(wr *os.File, size int64) error { // try contiguous first fst := unix.Fstore_t{ Flags: unix.F_ALLOCATECONTIG | unix.F_ALLOCATEALL, diff --git a/internal/restorer/preallocate_linux.go b/internal/fs/preallocate_linux.go similarity index 76% rename from internal/restorer/preallocate_linux.go rename to internal/fs/preallocate_linux.go index dc73ddfe2..30b9e4644 100644 --- a/internal/restorer/preallocate_linux.go +++ b/internal/fs/preallocate_linux.go @@ -1,4 +1,4 @@ -package restorer +package fs import ( "os" @@ -6,7 +6,7 @@ import ( "golang.org/x/sys/unix" ) -func preallocateFile(wr *os.File, size int64) error { +func PreallocateFile(wr *os.File, size int64) error { if size <= 0 { return nil } diff --git a/internal/restorer/preallocate_other.go b/internal/fs/preallocate_other.go similarity index 73% rename from internal/restorer/preallocate_other.go rename to internal/fs/preallocate_other.go index f01757bf4..4fb44d421 100644 --- a/internal/restorer/preallocate_other.go +++ b/internal/fs/preallocate_other.go @@ -1,11 +1,11 @@ //go:build !linux && !darwin // +build !linux,!darwin -package restorer +package fs import "os" -func preallocateFile(wr *os.File, size int64) error { +func PreallocateFile(wr *os.File, size int64) error { // Maybe truncate can help? // Windows: This calls SetEndOfFile which preallocates space on disk return wr.Truncate(size) diff --git a/internal/restorer/preallocate_test.go b/internal/fs/preallocate_test.go similarity index 85% rename from internal/restorer/preallocate_test.go rename to internal/fs/preallocate_test.go index 0cc2b3f5d..9dabd2f36 100644 --- a/internal/restorer/preallocate_test.go +++ b/internal/fs/preallocate_test.go @@ -1,4 +1,4 @@ -package restorer +package fs import ( "os" @@ -7,7 +7,6 @@ import ( "syscall" "testing" - "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/test" ) @@ -23,7 +22,7 @@ func TestPreallocate(t *testing.T) { test.OK(t, wr.Close()) }() - err = preallocateFile(wr, i) + err = PreallocateFile(wr, i) if err == syscall.ENOTSUP { t.SkipNow() } @@ -32,7 +31,7 @@ func TestPreallocate(t *testing.T) { fi, err := wr.Stat() test.OK(t, err) - efi := fs.ExtendedStat(fi) + efi := ExtendedStat(fi) test.Assert(t, efi.Size == i || efi.Blocks > 0, "Preallocated size of %v, got size %v block %v", i, efi.Size, efi.Blocks) }) } diff --git a/internal/restic/backend.go b/internal/restic/backend.go index b01071132..b6653fcb4 100644 --- a/internal/restic/backend.go +++ b/internal/restic/backend.go @@ -80,3 +80,8 @@ type FileInfo struct { Size int64 Name string } + +// ApplyEnvironmenter fills in a backend configuration from the environment +type ApplyEnvironmenter interface { + ApplyEnvironment(prefix string) error +} diff --git a/internal/restorer/fileswriter.go b/internal/restorer/fileswriter.go index 0a26101f4..589aa502a 100644 --- a/internal/restorer/fileswriter.go +++ b/internal/restorer/fileswriter.go @@ -6,6 +6,7 @@ import ( "github.com/cespare/xxhash/v2" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/fs" ) // writes blobs to target files. @@ -72,7 +73,7 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create return nil, err } } else { - err := preallocateFile(wr.File, createSize) + err := fs.PreallocateFile(wr.File, createSize) if err != nil { // Just log the preallocate error but don't let it cause the restore process to fail. // Preallocate might return an error if the filesystem (implementation) does not diff --git a/internal/ui/backup/progress.go b/internal/ui/backup/progress.go index 8e15662a6..4362a8c83 100644 --- a/internal/ui/backup/progress.go +++ b/internal/ui/backup/progress.go @@ -43,7 +43,8 @@ type Progress struct { progress.Updater mu sync.Mutex - start time.Time + start time.Time + estimator rateEstimator scanStarted, scanFinished bool @@ -60,6 +61,7 @@ func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress { start: time.Now(), currentFiles: make(map[string]struct{}), printer: printer, + estimator: *newRateEstimator(time.Now()), } p.Updater = *progress.NewUpdater(interval, func(runtime time.Duration, final bool) { if final { @@ -73,9 +75,14 @@ func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress { var secondsRemaining uint64 if p.scanFinished { - secs := float64(runtime / time.Second) - todo := float64(p.total.Bytes - p.processed.Bytes) - secondsRemaining = uint64(secs / float64(p.processed.Bytes) * todo) + rate := p.estimator.rate(time.Now()) + tooSlowCutoff := 1024. + if rate <= tooSlowCutoff { + secondsRemaining = 0 + } else { + todo := float64(p.total.Bytes - p.processed.Bytes) + secondsRemaining = uint64(todo / rate) + } } p.printer.Update(p.total, p.processed, p.errors, p.currentFiles, p.start, secondsRemaining) @@ -105,6 +112,7 @@ func (p *Progress) addProcessed(c Counter) { p.processed.Files += c.Files p.processed.Dirs += c.Dirs p.processed.Bytes += c.Bytes + p.estimator.recordBytes(time.Now(), c.Bytes) p.scanStarted = true } diff --git a/internal/ui/backup/rate_estimator.go b/internal/ui/backup/rate_estimator.go new file mode 100644 index 000000000..5291fbae1 --- /dev/null +++ b/internal/ui/backup/rate_estimator.go @@ -0,0 +1,98 @@ +package backup + +import ( + "container/list" + "time" +) + +// rateBucket represents a one second window of recorded progress. +type rateBucket struct { + totalBytes uint64 + end time.Time // the end of the time window, exclusive +} + +// rateEstimator represents an estimate of the time to complete an operation. +type rateEstimator struct { + buckets *list.List + start time.Time + totalBytes uint64 +} + +// newRateEstimator returns an estimator initialized to a presumed start time. +func newRateEstimator(start time.Time) *rateEstimator { + return &rateEstimator{buckets: list.New(), start: start} +} + +// See trim(), below. +const ( + bucketWidth = time.Second + minRateEstimatorBytes = 100 * 1000 * 1000 + minRateEstimatorBuckets = 20 + minRateEstimatorMinutes = 2 +) + +// trim removes the oldest history from the estimator assuming a given +// current time. +func (r *rateEstimator) trim(now time.Time) { + // The estimator retains byte transfer counts over a two minute window. + // However, to avoid removing too much history when transfer rates are + // low, the estimator also retains a minimum number of processed bytes + // across a minimum number of buckets. An operation that is processing a + // significant number of bytes per second will typically retain only a + // two minute window's worth of information. One that is making slow + // progress, such as one being over a rate limited connection, typically + // observes bursts of updates as infrequently as every ten or twenty + // seconds, in which case the other limiters will kick in. This heuristic + // avoids wildly fluctuating estimates over rate limited connections. + start := now.Add(-minRateEstimatorMinutes * time.Minute) + + for e := r.buckets.Front(); e != nil; e = r.buckets.Front() { + if r.buckets.Len() <= minRateEstimatorBuckets { + break + } + b := e.Value.(*rateBucket) + if b.end.After(start) { + break + } + total := r.totalBytes - b.totalBytes + if total < minRateEstimatorBytes { + break + } + r.start = b.end + r.totalBytes = total + r.buckets.Remove(e) + } +} + +// recordBytes records the transfer of a number of bytes at a given +// time. Times passed in successive calls should advance monotonically (as +// is the case with time.Now(). +func (r *rateEstimator) recordBytes(now time.Time, bytes uint64) { + if bytes == 0 { + return + } + var tail *rateBucket + if r.buckets.Len() > 0 { + tail = r.buckets.Back().Value.(*rateBucket) + } + if tail == nil || !tail.end.After(now) { + // The new bucket holds measurements in the time range [now .. now+1sec). + tail = &rateBucket{end: now.Add(bucketWidth)} + r.buckets.PushBack(tail) + } + tail.totalBytes += bytes + r.totalBytes += bytes + r.trim(now) +} + +// rate returns an estimated bytes per second rate at a given time, or zero +// if there is not enough data to compute a rate. +func (r *rateEstimator) rate(now time.Time) float64 { + r.trim(now) + if !r.start.Before(now) { + return 0 + } + elapsed := float64(now.Sub(r.start)) / float64(time.Second) + rate := float64(r.totalBytes) / elapsed + return rate +} diff --git a/internal/ui/backup/rate_estimator_test.go b/internal/ui/backup/rate_estimator_test.go new file mode 100644 index 000000000..0ebc6972b --- /dev/null +++ b/internal/ui/backup/rate_estimator_test.go @@ -0,0 +1,213 @@ +package backup + +import ( + "fmt" + "math" + "testing" + "time" + + rtest "github.com/restic/restic/internal/test" +) + +const float64EqualityThreshold = 1e-6 + +func almostEqual(a, b float64) bool { + if math.IsNaN(a) || math.IsNaN(b) { + panic("almostEqual passed a NaN") + } + return math.Abs(a-b) <= float64EqualityThreshold +} + +func TestEstimatorDefault(t *testing.T) { + var start time.Time + e := newRateEstimator(start) + r := e.rate(start) + rtest.Assert(t, r == 0, "e.Rate == %v, want zero", r) + r = e.rate(start.Add(time.Hour)) + rtest.Assert(t, r == 0, "e.Rate == %v, want zero", r) +} + +func TestEstimatorSimple(t *testing.T) { + var start time.Time + type testcase struct { + bytes uint64 + when time.Duration + rate float64 + } + + cases := []testcase{ + {0, 0, 0}, + {1, time.Second, 1}, + {60, time.Second, 60}, + {60, time.Minute, 1}, + } + for _, c := range cases { + name := fmt.Sprintf("%+v", c) + t.Run(name, func(t *testing.T) { + e := newRateEstimator(start) + e.recordBytes(start.Add(time.Second), c.bytes) + rate := e.rate(start.Add(c.when)) + rtest.Assert(t, almostEqual(rate, c.rate), "e.Rate == %v, want %v", rate, c.rate) + }) + } +} + +func TestBucketWidth(t *testing.T) { + var when time.Time + + // Recording byte transfers within a bucket width's time window uses one + // bucket. + e := newRateEstimator(when) + e.recordBytes(when, 1) + e.recordBytes(when.Add(bucketWidth-time.Nanosecond), 1) + rtest.Assert(t, e.buckets.Len() == 1, "e.buckets.Len() is %d, want 1", e.buckets.Len()) + + b := e.buckets.Back().Value.(*rateBucket) + rtest.Assert(t, b.totalBytes == 2, "b.totalBytes is %d, want 2", b.totalBytes) + rtest.Assert(t, b.end == when.Add(bucketWidth), "b.end is %v, want %v", b.end, when.Add(bucketWidth)) + + // Recording a byte outside the bucket width causes another bucket. + e.recordBytes(when.Add(bucketWidth), 1) + rtest.Assert(t, e.buckets.Len() == 2, "e.buckets.Len() is %d, want 2", e.buckets.Len()) + + b = e.buckets.Back().Value.(*rateBucket) + rtest.Assert(t, b.totalBytes == 1, "b.totalBytes is %d, want 1", b.totalBytes) + rtest.Assert(t, b.end == when.Add(2*bucketWidth), "b.end is %v, want %v", b.end, when.Add(bucketWidth)) + + // Recording a byte after a longer delay creates a sparse bucket list. + e.recordBytes(when.Add(time.Hour+time.Millisecond), 7) + rtest.Assert(t, e.buckets.Len() == 3, "e.buckets.Len() is %d, want 3", e.buckets.Len()) + + b = e.buckets.Back().Value.(*rateBucket) + rtest.Assert(t, b.totalBytes == 7, "b.totalBytes is %d, want 7", b.totalBytes) + rtest.Equals(t, when.Add(time.Hour+time.Millisecond+time.Second), b.end) +} + +type chunk struct { + repetitions uint64 // repetition count + bytes uint64 // byte count (every second) +} + +func applyChunks(chunks []chunk, t time.Time, e *rateEstimator) time.Time { + for _, c := range chunks { + for i := uint64(0); i < c.repetitions; i++ { + e.recordBytes(t, c.bytes) + t = t.Add(time.Second) + } + } + return t +} + +func TestEstimatorResponsiveness(t *testing.T) { + type testcase struct { + description string + chunks []chunk + rate float64 + } + + cases := []testcase{ + { + "1000 bytes/sec over one second", + []chunk{ + {1, 1000}, + }, + 1000, + }, + { + "1000 bytes/sec over one minute", + []chunk{ + {60, 1000}, + }, + 1000, + }, + { + "1000 bytes/sec for 10 seconds, then 2000 bytes/sec for 10 seconds", + []chunk{ + {10, 1000}, + {10, 2000}, + }, + 1500, + }, + { + "1000 bytes/sec for one minute, then 2000 bytes/sec for one minute", + []chunk{ + {60, 1000}, + {60, 2000}, + }, + 1500, + }, + { + "rate doubles after 30 seconds", + []chunk{ + {30, minRateEstimatorBytes}, + {90, 2 * minRateEstimatorBytes}, + }, + minRateEstimatorBytes * 1.75, + }, + { + "rate doubles after 31 seconds", + []chunk{ + {31, minRateEstimatorBytes}, + {90, 2 * minRateEstimatorBytes}, + }, + // The expected rate is the same as the prior test case because the + // first second has rolled off the estimator. + minRateEstimatorBytes * 1.75, + }, + { + "rate doubles after 90 seconds", + []chunk{ + {90, minRateEstimatorBytes}, + {90, 2 * minRateEstimatorBytes}, + }, + // The expected rate is the same as the prior test case because the + // first 60 seconds have rolled off the estimator. + minRateEstimatorBytes * 1.75, + }, + { + "rate doubles for two full minutes", + []chunk{ + {60, minRateEstimatorBytes}, + {120, 2 * minRateEstimatorBytes}, + }, + 2 * minRateEstimatorBytes, + }, + { + "rate falls to zero", + []chunk{ + {30, minRateEstimatorBytes}, + {30, 0}, + }, + minRateEstimatorBytes / 2, + }, + { + "rate falls to zero for extended time", + []chunk{ + {60, 1000}, + {300, 0}, + }, + 1000 * 60 / (60 + 300.0), + }, + { + "rate falls to zero for extended time (from high rate)", + []chunk{ + {2 * minRateEstimatorBuckets, minRateEstimatorBytes}, + {300, 0}, + }, + // Expect that only minRateEstimatorBuckets buckets are used in the + // rate estimate. + minRateEstimatorBytes * minRateEstimatorBuckets / + (minRateEstimatorBuckets + 300.0), + }, + } + + for _, c := range cases { + t.Run(c.description, func(t *testing.T) { + var w time.Time + e := newRateEstimator(w) + w = applyChunks(c.chunks, w, e) + r := e.rate(w) + rtest.Assert(t, almostEqual(r, c.rate), "e.Rate == %f, want %f", r, c.rate) + }) + } +}