Mirrors/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2025-03-16 00:00:05 +01:00
Code Issues Releases Wiki Activity

Merge remote-tracking branch 'old-origin/master' into add-smb-backend

Browse source
This commit is contained in:
Aneesh Nireshwalia 2023-07-10 11:17:50 -06:00
commit 4275870876
50 changed files with 966 additions and 758 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
changelog/unreleased/issue-3397 Normal file
View file

@ -0,0 +1,11 @@
Enhancement: Improve the ETA displayed during backup
Restic's `backup` command displayed an ETA that did not adapt when the rate
of progress made during the backup changed during the course of the
backup. Restic now uses recent progress when computing the ETA. It is
important to realize that the estimate may still be wrong, because restic
cannot predict the future, but the hope is that the ETA will be more
accurate in most cases.
https://github.com/restic/restic/issues/3397
https://github.com/restic/restic/pull/3563

8
changelog/unreleased/pull-3261 Normal file
View file

@ -0,0 +1,8 @@
Enhancement: Reduce file fragmentation for local backend
Before this change, local backend files could become fragmented.
Now restic will try to preallocate space for pack files to avoid
their fragmentation.
https://github.com/restic/restic/issues/2679
https://github.com/restic/restic/pull/3261

211
cmd/restic/global.go
View file

@ -536,170 +536,21 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi
}
func parseConfig(loc location.Location, opts options.Options) (interface{}, error) {
cfg := loc.Config
if cfg, ok := cfg.(restic.ApplyEnvironmenter); ok {
if err := cfg.ApplyEnvironment(""); err != nil {
return nil, err
}
}
// only apply options for a particular backend here
opts = opts.Extract(loc.Scheme)
switch loc.Scheme {
case "local":
cfg := loc.Config.(local.Config)
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
if err := opts.Apply(loc.Scheme, cfg); err != nil {
return nil, err
}
debug.Log("opening local repository at %#v", cfg)
debug.Log("opening %v repository at %#v", loc.Scheme, cfg)
return cfg, nil
case "sftp":
cfg := loc.Config.(sftp.Config)
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
return nil, err
}
debug.Log("opening sftp repository at %#v", cfg)
return cfg, nil
case "s3":
cfg := loc.Config.(s3.Config)
if cfg.KeyID == "" {
cfg.KeyID = os.Getenv("AWS_ACCESS_KEY_ID")
}
if cfg.Secret.String() == "" {
cfg.Secret = options.NewSecretString(os.Getenv("AWS_SECRET_ACCESS_KEY"))
}
if cfg.KeyID == "" && cfg.Secret.String() != "" {
return nil, errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty")
} else if cfg.KeyID != "" && cfg.Secret.String() == "" {
return nil, errors.Fatalf("unable to open S3 backend: Secret ($AWS_SECRET_ACCESS_KEY) is empty")
}
if cfg.Region == "" {
cfg.Region = os.Getenv("AWS_DEFAULT_REGION")
}
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
return nil, err
}
debug.Log("opening s3 repository at %#v", cfg)
return cfg, nil
case "gs":
cfg := loc.Config.(gs.Config)
if cfg.ProjectID == "" {
cfg.ProjectID = os.Getenv("GOOGLE_PROJECT_ID")
}
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
return nil, err
}
debug.Log("opening gs repository at %#v", cfg)
return cfg, nil
case "azure":
cfg := loc.Config.(azure.Config)
if cfg.AccountName == "" {
cfg.AccountName = os.Getenv("AZURE_ACCOUNT_NAME")
}
if cfg.AccountKey.String() == "" {
cfg.AccountKey = options.NewSecretString(os.Getenv("AZURE_ACCOUNT_KEY"))
}
if cfg.AccountSAS.String() == "" {
cfg.AccountSAS = options.NewSecretString(os.Getenv("AZURE_ACCOUNT_SAS"))
}
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
return nil, err
}
debug.Log("opening gs repository at %#v", cfg)
return cfg, nil
case "swift":
cfg := loc.Config.(swift.Config)
if err := swift.ApplyEnvironment("", &cfg); err != nil {
return nil, err
}
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
return nil, err
}
debug.Log("opening swift repository at %#v", cfg)
return cfg, nil
case "b2":
cfg := loc.Config.(b2.Config)
if cfg.AccountID == "" {
cfg.AccountID = os.Getenv("B2_ACCOUNT_ID")
}
if cfg.AccountID == "" {
return nil, errors.Fatalf("unable to open B2 backend: Account ID ($B2_ACCOUNT_ID) is empty")
}
if cfg.Key.String() == "" {
cfg.Key = options.NewSecretString(os.Getenv("B2_ACCOUNT_KEY"))
}
if cfg.Key.String() == "" {
return nil, errors.Fatalf("unable to open B2 backend: Key ($B2_ACCOUNT_KEY) is empty")
}
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
return nil, err
}
debug.Log("opening b2 repository at %#v", cfg)
return cfg, nil
case "rest":
cfg := loc.Config.(rest.Config)
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
return nil, err
}
debug.Log("opening rest repository at %#v", cfg)
return cfg, nil
case "rclone":
cfg := loc.Config.(rclone.Config)
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
return nil, err
}
debug.Log("opening rest repository at %#v", cfg)
return cfg, nil
case "smb":
cfg := loc.Config.(smb.Config)
if cfg.User == "" {
cfg.User = os.Getenv("RESTIC_SMB_USER")
}
if cfg.Password.String() == "" {
cfg.Password = options.NewSecretString(os.Getenv("RESTIC_SMB_PASSWORD"))
}
if cfg.Domain == "" {
cfg.Domain = os.Getenv("RESTIC_SMB_DOMAIN")
}
if cfg.Domain == "" {
cfg.Domain = smb.DefaultDomain
}
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
return nil, err
}
debug.Log("opening smb repository at %#v", cfg)
return cfg, nil
}
return nil, errors.Fatalf("invalid backend: %q", loc.Scheme)
}
// Open the backend specified by a location config.
@ -728,25 +579,25 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio
switch loc.Scheme {
case "local":
be, err = local.Open(ctx, cfg.(local.Config))
be, err = local.Open(ctx, *cfg.(*local.Config))
case "sftp":
be, err = sftp.Open(ctx, cfg.(sftp.Config))
case "smb":
be, err = smb.Open(ctx, cfg.(smb.Config))
be, err = sftp.Open(ctx, *cfg.(*sftp.Config))
case "s3":
be, err = s3.Open(ctx, cfg.(s3.Config), rt)
be, err = s3.Open(ctx, *cfg.(*s3.Config), rt)
case "gs":
be, err = gs.Open(cfg.(gs.Config), rt)
be, err = gs.Open(*cfg.(*gs.Config), rt)
case "azure":
be, err = azure.Open(ctx, cfg.(azure.Config), rt)
be, err = azure.Open(ctx, *cfg.(*azure.Config), rt)
case "swift":
be, err = swift.Open(ctx, cfg.(swift.Config), rt)
be, err = swift.Open(ctx, *cfg.(*swift.Config), rt)
case "b2":
be, err = b2.Open(ctx, cfg.(b2.Config), rt)
be, err = b2.Open(ctx, *cfg.(*b2.Config), rt)
case "rest":
be, err = rest.Open(cfg.(rest.Config), rt)
be, err = rest.Open(*cfg.(*rest.Config), rt)
case "rclone":
be, err = rclone.Open(cfg.(rclone.Config), lim)
be, err = rclone.Open(*cfg.(*rclone.Config), lim)
case "smb":
be, err = smb.Open(ctx, *cfg.(*smb.Config))
default:
return nil, errors.Fatalf("invalid backend: %q", loc.Scheme)
@ -806,25 +657,25 @@ func create(ctx context.Context, s string, opts options.Options) (restic.Backend
var be restic.Backend
switch loc.Scheme {
case "local":
be, err = local.Create(ctx, cfg.(local.Config))
be, err = local.Create(ctx, *cfg.(*local.Config))
case "sftp":
be, err = sftp.Create(ctx, cfg.(sftp.Config))
case "smb":
be, err = smb.Create(ctx, cfg.(smb.Config))
be, err = sftp.Create(ctx, *cfg.(*sftp.Config))
case "s3":
be, err = s3.Create(ctx, cfg.(s3.Config), rt)
be, err = s3.Create(ctx, *cfg.(*s3.Config), rt)
case "gs":
be, err = gs.Create(ctx, cfg.(gs.Config), rt)
be, err = gs.Create(ctx, *cfg.(*gs.Config), rt)
case "azure":
be, err = azure.Create(ctx, cfg.(azure.Config), rt)
be, err = azure.Create(ctx, *cfg.(*azure.Config), rt)
case "swift":
be, err = swift.Open(ctx, cfg.(swift.Config), rt)
be, err = swift.Open(ctx, *cfg.(*swift.Config), rt)
case "b2":
be, err = b2.Create(ctx, cfg.(b2.Config), rt)
be, err = b2.Create(ctx, *cfg.(*b2.Config), rt)
case "rest":
be, err = rest.Create(ctx, cfg.(rest.Config), rt)
be, err = rest.Create(ctx, *cfg.(*rest.Config), rt)
case "rclone":
be, err = rclone.Create(ctx, cfg.(rclone.Config))
be, err = rclone.Create(ctx, *cfg.(*rclone.Config))
case "smb":
be, err = smb.Create(ctx, *cfg.(*smb.Config))
default:
debug.Log("invalid repository scheme: %v", s)
return nil, errors.Fatalf("invalid scheme %q", loc.Scheme)

31
internal/backend/azure/azure_test.go
View file

@ -18,34 +18,34 @@ import (
rtest "github.com/restic/restic/internal/test"
)
func newAzureTestSuite(t testing.TB) *test.Suite {
func newAzureTestSuite(t testing.TB) *test.Suite[azure.Config] {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
return &test.Suite{
return &test.Suite[azure.Config]{
// do not use excessive data
MinimalData: true,
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
azcfg, err := azure.ParseConfig(os.Getenv("RESTIC_TEST_AZURE_REPOSITORY"))
NewConfig: func() (*azure.Config, error) {
cfg, err := azure.ParseConfig(os.Getenv("RESTIC_TEST_AZURE_REPOSITORY"))
if err != nil {
return nil, err
}
err = cfg.ApplyEnvironment("RESTIC_TEST_")
if err != nil {
return nil, err
}
cfg := azcfg.(azure.Config)
cfg.AccountName = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_NAME")
cfg.AccountKey = options.NewSecretString(os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY"))
cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
return cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(azure.Config)
Create: func(cfg azure.Config) (restic.Backend, error) {
ctx := context.TODO()
be, err := azure.Create(ctx, cfg, tr)
if err != nil {
@ -65,15 +65,13 @@ func newAzureTestSuite(t testing.TB) *test.Suite {
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(azure.Config)
Open: func(cfg azure.Config) (restic.Backend, error) {
ctx := context.TODO()
return azure.Open(ctx, cfg, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
cfg := config.(azure.Config)
Cleanup: func(cfg azure.Config) error {
ctx := context.TODO()
be, err := azure.Open(ctx, cfg, tr)
if err != nil {
@ -141,12 +139,11 @@ func TestUploadLargeFile(t *testing.T) {
return
}
azcfg, err := azure.ParseConfig(os.Getenv("RESTIC_TEST_AZURE_REPOSITORY"))
cfg, err := azure.ParseConfig(os.Getenv("RESTIC_TEST_AZURE_REPOSITORY"))
if err != nil {
t.Fatal(err)
}
cfg := azcfg.(azure.Config)
cfg.AccountName = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_NAME")
cfg.AccountKey = options.NewSecretString(os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY"))
cfg.Prefix = fmt.Sprintf("test-upload-large-%d", time.Now().UnixNano())
@ -156,7 +153,7 @@ func TestUploadLargeFile(t *testing.T) {
t.Fatal(err)
}
be, err := azure.Create(ctx, cfg, tr)
be, err := azure.Create(ctx, *cfg, tr)
if err != nil {
t.Fatal(err)
}

24
internal/backend/azure/config.go
View file

@ -1,11 +1,13 @@
package azure
import (
"os"
"path"
"strings"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains all configuration necessary to connect to an azure compatible
@ -33,7 +35,7 @@ func init() {
// ParseConfig parses the string s and extracts the azure config. The
// configuration format is azure:containerName:/[prefix].
func ParseConfig(s string) (interface{}, error) {
func ParseConfig(s string) (*Config, error) {
if !strings.HasPrefix(s, "azure:") {
return nil, errors.New("azure: invalid format")
}
@ -51,5 +53,23 @@ func ParseConfig(s string) (interface{}, error) {
cfg := NewConfig()
cfg.Container = container
cfg.Prefix = prefix
return cfg, nil
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) error {
if cfg.AccountName == "" {
cfg.AccountName = os.Getenv(prefix + "AZURE_ACCOUNT_NAME")
}
if cfg.AccountKey.String() == "" {
cfg.AccountKey = options.NewSecretString(os.Getenv(prefix + "AZURE_ACCOUNT_KEY"))
}
if cfg.AccountSAS.String() == "" {
cfg.AccountSAS = options.NewSecretString(os.Getenv(prefix + "AZURE_ACCOUNT_SAS"))
}
return nil
}

31
internal/backend/azure/config_test.go
View file

@ -1,22 +1,23 @@
package azure
import "testing"
import (
"testing"
var configTests = []struct {
s string
cfg Config
}{
{"azure:container-name:/", Config{
"github.com/restic/restic/internal/backend/test"
)
var configTests = []test.ConfigTestData[Config]{
{S: "azure:container-name:/", Cfg: Config{
Container: "container-name",
Prefix: "",
Connections: 5,
}},
{"azure:container-name:/prefix/directory", Config{
{S: "azure:container-name:/prefix/directory", Cfg: Config{
Container: "container-name",
Prefix: "prefix/directory",
Connections: 5,
}},
{"azure:container-name:/prefix/directory/", Config{
{S: "azure:container-name:/prefix/directory/", Cfg: Config{
Container: "container-name",
Prefix: "prefix/directory",
Connections: 5,
@ -24,17 +25,5 @@ var configTests = []struct {
}
func TestParseConfig(t *testing.T) {
for i, test := range configTests {
cfg, err := ParseConfig(test.s)
if err != nil {
t.Errorf("test %d:%s failed: %v", i, test.s, err)
continue
}
if cfg != test.cfg {
t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v",
i, test.s, test.cfg, cfg)
continue
}
}
test.ParseConfigTester(t, ParseConfig, configTests)
}

26
internal/backend/b2/b2_test.go
View file

@ -10,19 +10,18 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/b2"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func newB2TestSuite(t testing.TB) *test.Suite {
func newB2TestSuite(t testing.TB) *test.Suite[b2.Config] {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
return &test.Suite{
return &test.Suite[b2.Config]{
// do not use excessive data
MinimalData: true,
@ -30,34 +29,33 @@ func newB2TestSuite(t testing.TB) *test.Suite {
WaitForDelayedRemoval: 10 * time.Second,
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
b2cfg, err := b2.ParseConfig(os.Getenv("RESTIC_TEST_B2_REPOSITORY"))
NewConfig: func() (*b2.Config, error) {
cfg, err := b2.ParseConfig(os.Getenv("RESTIC_TEST_B2_REPOSITORY"))
if err != nil {
return nil, err
}
err = cfg.ApplyEnvironment("RESTIC_TEST_")
if err != nil {
return nil, err
}
cfg := b2cfg.(b2.Config)
cfg.AccountID = os.Getenv("RESTIC_TEST_B2_ACCOUNT_ID")
cfg.Key = options.NewSecretString(os.Getenv("RESTIC_TEST_B2_ACCOUNT_KEY"))
cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
return cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(b2.Config)
Create: func(cfg b2.Config) (restic.Backend, error) {
return b2.Create(context.Background(), cfg, tr)
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(b2.Config)
Open: func(cfg b2.Config) (restic.Backend, error) {
return b2.Open(context.Background(), cfg, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
cfg := config.(b2.Config)
Cleanup: func(cfg b2.Config) error {
be, err := b2.Open(context.Background(), cfg, tr)
if err != nil {
return err

28
internal/backend/b2/config.go
View file

@ -1,12 +1,14 @@
package b2
import (
"os"
"path"
"regexp"
"strings"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains all configuration necessary to connect to an b2 compatible
@ -58,7 +60,7 @@ func checkBucketName(name string) error {
// ParseConfig parses the string s and extracts the b2 config. The supported
// configuration format is b2:bucketname/prefix. If no prefix is given the
// prefix "restic" will be used.
func ParseConfig(s string) (interface{}, error) {
func ParseConfig(s string) (*Config, error) {
if !strings.HasPrefix(s, "b2:") {
return nil, errors.New("invalid format, want: b2:bucket-name[:path]")
}
@ -77,5 +79,27 @@ func ParseConfig(s string) (interface{}, error) {
cfg.Bucket = bucket
cfg.Prefix = prefix
return cfg, nil
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) error {
if cfg.AccountID == "" {
cfg.AccountID = os.Getenv(prefix + "B2_ACCOUNT_ID")
}
if cfg.AccountID == "" {
return errors.Fatalf("unable to open B2 backend: Account ID ($B2_ACCOUNT_ID) is empty")
}
if cfg.Key.String() == "" {
cfg.Key = options.NewSecretString(os.Getenv(prefix + "B2_ACCOUNT_KEY"))
}
if cfg.Key.String() == "" {
return errors.Fatalf("unable to open B2 backend: Key ($B2_ACCOUNT_KEY) is empty")
}
return nil
}

37
internal/backend/b2/config_test.go
View file

@ -1,37 +1,38 @@
package b2
import "testing"
import (
"testing"
var configTests = []struct {
s string
cfg Config
}{
{"b2:bucketname", Config{
"github.com/restic/restic/internal/backend/test"
)
var configTests = []test.ConfigTestData[Config]{
{S: "b2:bucketname", Cfg: Config{
Bucket: "bucketname",
Prefix: "",
Connections: 5,
}},
{"b2:bucketname:", Config{
{S: "b2:bucketname:", Cfg: Config{
Bucket: "bucketname",
Prefix: "",
Connections: 5,
}},
{"b2:bucketname:/prefix/directory", Config{
{S: "b2:bucketname:/prefix/directory", Cfg: Config{
Bucket: "bucketname",
Prefix: "prefix/directory",
Connections: 5,
}},
{"b2:foobar", Config{
{S: "b2:foobar", Cfg: Config{
Bucket: "foobar",
Prefix: "",
Connections: 5,
}},
{"b2:foobar:", Config{
{S: "b2:foobar:", Cfg: Config{
Bucket: "foobar",
Prefix: "",
Connections: 5,
}},
{"b2:foobar:/", Config{
{S: "b2:foobar:/", Cfg: Config{
Bucket: "foobar",
Prefix: "",
Connections: 5,
@ -39,19 +40,7 @@ var configTests = []struct {
}
func TestParseConfig(t *testing.T) {
for _, test := range configTests {
t.Run("", func(t *testing.T) {
cfg, err := ParseConfig(test.s)
if err != nil {
t.Fatalf("%s failed: %v", test.s, err)
}
if cfg != test.cfg {
t.Fatalf("input: %s\n wrong config, want:\n %#v\ngot:\n %#v",
test.s, test.cfg, cfg)
}
})
}
test.ParseConfigTester(t, ParseConfig, configTests)
}
var invalidConfigTests = []struct {

16
internal/backend/gs/config.go
View file

@ -1,11 +1,13 @@
package gs
import (
"os"
"path"
"strings"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains all configuration necessary to connect to a Google Cloud Storage
@ -34,7 +36,7 @@ func init() {
// ParseConfig parses the string s and extracts the gcs config. The
// supported configuration format is gs:bucketName:/[prefix].
func ParseConfig(s string) (interface{}, error) {
func ParseConfig(s string) (*Config, error) {
if !strings.HasPrefix(s, "gs:") {
return nil, errors.New("gs: invalid format")
}
@ -54,5 +56,15 @@ func ParseConfig(s string) (interface{}, error) {
cfg := NewConfig()
cfg.Bucket = bucket
cfg.Prefix = prefix
return cfg, nil
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) error {
if cfg.ProjectID == "" {
cfg.ProjectID = os.Getenv(prefix + "GOOGLE_PROJECT_ID")
}
return nil
}

31
internal/backend/gs/config_test.go
View file

@ -1,24 +1,25 @@
package gs
import "testing"
import (
"testing"
var configTests = []struct {
s string
cfg Config
}{
{"gs:bucketname:/", Config{
"github.com/restic/restic/internal/backend/test"
)
var configTests = []test.ConfigTestData[Config]{
{S: "gs:bucketname:/", Cfg: Config{
Bucket: "bucketname",
Prefix: "",
Connections: 5,
Region: "us",
}},
{"gs:bucketname:/prefix/directory", Config{
{S: "gs:bucketname:/prefix/directory", Cfg: Config{
Bucket: "bucketname",
Prefix: "prefix/directory",
Connections: 5,
Region: "us",
}},
{"gs:bucketname:/prefix/directory/", Config{
{S: "gs:bucketname:/prefix/directory/", Cfg: Config{
Bucket: "bucketname",
Prefix: "prefix/directory",
Connections: 5,
@ -27,17 +28,5 @@ var configTests = []struct {
}
func TestParseConfig(t *testing.T) {
for i, test := range configTests {
cfg, err := ParseConfig(test.s)
if err != nil {
t.Errorf("test %d:%s failed: %v", i, test.s, err)
continue
}
if cfg != test.cfg {
t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v",
i, test.s, test.cfg, cfg)
continue
}
}
test.ParseConfigTester(t, ParseConfig, configTests)
}

20
internal/backend/gs/gs_test.go
View file

@ -15,33 +15,30 @@ import (
rtest "github.com/restic/restic/internal/test"
)
func newGSTestSuite(t testing.TB) *test.Suite {
func newGSTestSuite(t testing.TB) *test.Suite[gs.Config] {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
return &test.Suite{
return &test.Suite[gs.Config]{
// do not use excessive data
MinimalData: true,
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
gscfg, err := gs.ParseConfig(os.Getenv("RESTIC_TEST_GS_REPOSITORY"))
NewConfig: func() (*gs.Config, error) {
cfg, err := gs.ParseConfig(os.Getenv("RESTIC_TEST_GS_REPOSITORY"))
if err != nil {
return nil, err
}
cfg := gscfg.(gs.Config)
cfg.ProjectID = os.Getenv("RESTIC_TEST_GS_PROJECT_ID")
cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
return cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(gs.Config)
Create: func(cfg gs.Config) (restic.Backend, error) {
be, err := gs.Create(context.Background(), cfg, tr)
if err != nil {
return nil, err
@ -60,15 +57,12 @@ func newGSTestSuite(t testing.TB) *test.Suite {
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(gs.Config)
Open: func(cfg gs.Config) (restic.Backend, error) {
return gs.Open(cfg, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
cfg := config.(gs.Config)
Cleanup: func(cfg gs.Config) error {
be, err := gs.Open(cfg, tr)
if err != nil {
return err

4
internal/backend/local/config.go
View file

@ -27,12 +27,12 @@ func init() {
}
// ParseConfig parses a local backend config.
func ParseConfig(s string) (interface{}, error) {
func ParseConfig(s string) (*Config, error) {
if !strings.HasPrefix(s, "local:") {
return nil, errors.New(`invalid format, prefix "local" not found`)
}
cfg := NewConfig()
cfg.Path = s[6:]
return cfg, nil
return &cfg, nil
}

18
internal/backend/local/config_test.go Normal file
View file

@ -0,0 +1,18 @@
package local
import (
"testing"
"github.com/restic/restic/internal/backend/test"
)
var configTests = []test.ConfigTestData[Config]{
{S: "local:/some/path", Cfg: Config{
Path: "/some/path",
Connections: 2,
}},
}
func TestParseConfig(t *testing.T) {
test.ParseConfigTester(t, ParseConfig, configTests)
}

7
internal/backend/local/local.go
View file

@ -148,6 +148,13 @@ func (b *Local) Save(_ context.Context, h restic.Handle, rd restic.RewindReader)
}
}(f)
// preallocate disk space
if size := rd.Length(); size > 0 {
if err := fs.PreallocateFile(f, size); err != nil {
debug.Log("Failed to preallocate %v with size %v: %v", finalname, size, err)
}
}
// save data, then sync
wbytes, err := io.Copy(f, rd)
if err != nil {

17
internal/backend/local/local_test.go
View file

@ -12,10 +12,10 @@ import (
rtest "github.com/restic/restic/internal/test"
)
func newTestSuite(t testing.TB) *test.Suite {
return &test.Suite{
func newTestSuite(t testing.TB) *test.Suite[local.Config] {
return &test.Suite[local.Config]{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
NewConfig: func() (*local.Config, error) {
dir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-local-")
if err != nil {
t.Fatal(err)
@ -23,7 +23,7 @@ func newTestSuite(t testing.TB) *test.Suite {
t.Logf("create new backend at %v", dir)
cfg := local.Config{
cfg := &local.Config{
Path: dir,
Connections: 2,
}
@ -31,20 +31,17 @@ func newTestSuite(t testing.TB) *test.Suite {
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(local.Config)
Create: func(cfg local.Config) (restic.Backend, error) {
return local.Create(context.TODO(), cfg)
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(local.Config)
Open: func(cfg local.Config) (restic.Backend, error) {
return local.Open(context.TODO(), cfg)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
cfg := config.(local.Config)
Cleanup: func(cfg local.Config) error {
if !rtest.TestCleanupTempDirs {
t.Logf("leaving test backend dir at %v", cfg.Path)
}

26
internal/backend/location/location.go
View file

@ -30,19 +30,25 @@ type parser struct {
stripPassword func(string) string
}
func configToAny[C any](parser func(string) (*C, error)) func(string) (interface{}, error) {
return func(s string) (interface{}, error) {
return parser(s)
}
}
// parsers is a list of valid config parsers for the backends. The first parser
// is the fallback and should always be set to the local backend.
var parsers = []parser{
{"b2", b2.ParseConfig, noPassword},
{"local", local.ParseConfig, noPassword},
{"sftp", sftp.ParseConfig, noPassword},
{"s3", s3.ParseConfig, noPassword},
{"gs", gs.ParseConfig, noPassword},
{"azure", azure.ParseConfig, noPassword},
{"swift", swift.ParseConfig, noPassword},
{"rest", rest.ParseConfig, rest.StripPassword},
{"rclone", rclone.ParseConfig, noPassword},
{"smb", smb.ParseConfig, noPassword},
{"b2", configToAny(b2.ParseConfig), noPassword},
{"local", configToAny(local.ParseConfig), noPassword},
{"sftp", configToAny(sftp.ParseConfig), noPassword},
{"s3", configToAny(s3.ParseConfig), noPassword},
{"gs", configToAny(gs.ParseConfig), noPassword},
{"azure", configToAny(azure.ParseConfig), noPassword},
{"swift", configToAny(swift.ParseConfig), noPassword},
{"rest", configToAny(rest.ParseConfig), rest.StripPassword},
{"rclone", configToAny(rclone.ParseConfig), noPassword},
{"smb", configToAny(smb.ParseConfig), noPassword},
}
// noPassword returns the repository location unchanged (there's no sensitive information there)

58
internal/backend/location/location_test.go
View file

@ -29,7 +29,7 @@ var parseTests = []struct {
{
"local:/srv/repo",
Location{Scheme: "local",
Config: local.Config{
Config: &local.Config{
Path: "/srv/repo",
Connections: 2,
},
@ -38,7 +38,7 @@ var parseTests = []struct {
{
"local:dir1/dir2",
Location{Scheme: "local",
Config: local.Config{
Config: &local.Config{
Path: "dir1/dir2",
Connections: 2,
},
@ -47,7 +47,7 @@ var parseTests = []struct {
{
"local:dir1/dir2",
Location{Scheme: "local",
Config: local.Config{
Config: &local.Config{
Path: "dir1/dir2",
Connections: 2,
},
@ -56,7 +56,7 @@ var parseTests = []struct {
{
"dir1/dir2",
Location{Scheme: "local",
Config: local.Config{
Config: &local.Config{
Path: "dir1/dir2",
Connections: 2,
},
@ -65,7 +65,7 @@ var parseTests = []struct {
{
"/dir1/dir2",
Location{Scheme: "local",
Config: local.Config{
Config: &local.Config{
Path: "/dir1/dir2",
Connections: 2,
},
@ -74,7 +74,7 @@ var parseTests = []struct {
{
"local:../dir1/dir2",
Location{Scheme: "local",
Config: local.Config{
Config: &local.Config{
Path: "../dir1/dir2",
Connections: 2,
},
@ -83,7 +83,7 @@ var parseTests = []struct {
{
"/dir1/dir2",
Location{Scheme: "local",
Config: local.Config{
Config: &local.Config{
Path: "/dir1/dir2",
Connections: 2,
},
@ -92,7 +92,7 @@ var parseTests = []struct {
{
"/dir1:foobar/dir2",
Location{Scheme: "local",
Config: local.Config{
Config: &local.Config{
Path: "/dir1:foobar/dir2",
Connections: 2,
},
@ -101,7 +101,7 @@ var parseTests = []struct {
{
`\dir1\foobar\dir2`,
Location{Scheme: "local",
Config: local.Config{
Config: &local.Config{
Path: `\dir1\foobar\dir2`,
Connections: 2,
},
@ -110,7 +110,7 @@ var parseTests = []struct {
{
`c:\dir1\foobar\dir2`,
Location{Scheme: "local",
Config: local.Config{
Config: &local.Config{
Path: `c:\dir1\foobar\dir2`,
Connections: 2,
},
@ -119,7 +119,7 @@ var parseTests = []struct {
{
`C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`,
Location{Scheme: "local",
Config: local.Config{
Config: &local.Config{
Path: `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`,
Connections: 2,
},
@ -128,7 +128,7 @@ var parseTests = []struct {
{
`c:/dir1/foobar/dir2`,
Location{Scheme: "local",
Config: local.Config{
Config: &local.Config{
Path: `c:/dir1/foobar/dir2`,
Connections: 2,
},
@ -137,7 +137,7 @@ var parseTests = []struct {
{
"sftp:user@host:/srv/repo",
Location{Scheme: "sftp",
Config: sftp.Config{
Config: &sftp.Config{
User: "user",
Host: "host",
Path: "/srv/repo",
@ -148,7 +148,7 @@ var parseTests = []struct {
{
"sftp:host:/srv/repo",
Location{Scheme: "sftp",
Config: sftp.Config{
Config: &sftp.Config{
User: "",
Host: "host",
Path: "/srv/repo",
@ -159,7 +159,7 @@ var parseTests = []struct {
{
"sftp://user@host/srv/repo",
Location{Scheme: "sftp",
Config: sftp.Config{
Config: &sftp.Config{
User: "user",
Host: "host",
Path: "srv/repo",
@ -170,7 +170,7 @@ var parseTests = []struct {
{
"sftp://user@host//srv/repo",
Location{Scheme: "sftp",
Config: sftp.Config{
Config: &sftp.Config{
User: "user",
Host: "host",
Path: "/srv/repo",
@ -182,7 +182,7 @@ var parseTests = []struct {
{
"s3://eu-central-1/bucketname",
Location{Scheme: "s3",
Config: s3.Config{
Config: &s3.Config{
Endpoint: "eu-central-1",
Bucket: "bucketname",
Prefix: "",
@ -193,7 +193,7 @@ var parseTests = []struct {
{
"s3://hostname.foo/bucketname",
Location{Scheme: "s3",
Config: s3.Config{
Config: &s3.Config{
Endpoint: "hostname.foo",
Bucket: "bucketname",
Prefix: "",
@ -204,7 +204,7 @@ var parseTests = []struct {
{
"s3://hostname.foo/bucketname/prefix/directory",
Location{Scheme: "s3",
Config: s3.Config{
Config: &s3.Config{
Endpoint: "hostname.foo",
Bucket: "bucketname",
Prefix: "prefix/directory",
@ -215,7 +215,7 @@ var parseTests = []struct {
{
"s3:eu-central-1/repo",
Location{Scheme: "s3",
Config: s3.Config{
Config: &s3.Config{
Endpoint: "eu-central-1",
Bucket: "repo",
Prefix: "",
@ -226,7 +226,7 @@ var parseTests = []struct {
{
"s3:eu-central-1/repo/prefix/directory",
Location{Scheme: "s3",
Config: s3.Config{
Config: &s3.Config{
Endpoint: "eu-central-1",
Bucket: "repo",
Prefix: "prefix/directory",
@ -237,7 +237,7 @@ var parseTests = []struct {
{
"s3:https://hostname.foo/repo",
Location{Scheme: "s3",
Config: s3.Config{
Config: &s3.Config{
Endpoint: "hostname.foo",
Bucket: "repo",
Prefix: "",
@ -248,7 +248,7 @@ var parseTests = []struct {
{
"s3:https://hostname.foo/repo/prefix/directory",
Location{Scheme: "s3",
Config: s3.Config{
Config: &s3.Config{
Endpoint: "hostname.foo",
Bucket: "repo",
Prefix: "prefix/directory",
@ -259,7 +259,7 @@ var parseTests = []struct {
{
"s3:http://hostname.foo/repo",
Location{Scheme: "s3",
Config: s3.Config{
Config: &s3.Config{
Endpoint: "hostname.foo",
Bucket: "repo",
Prefix: "",
@ -271,7 +271,7 @@ var parseTests = []struct {
{
"swift:container17:/",
Location{Scheme: "swift",
Config: swift.Config{
Config: &swift.Config{
Container: "container17",
Prefix: "",
Connections: 5,
@ -281,7 +281,7 @@ var parseTests = []struct {
{
"swift:container17:/prefix97",
Location{Scheme: "swift",
Config: swift.Config{
Config: &swift.Config{
Container: "container17",
Prefix: "prefix97",
Connections: 5,
@ -291,7 +291,7 @@ var parseTests = []struct {
{
"rest:http://hostname.foo:1234/",
Location{Scheme: "rest",
Config: rest.Config{
Config: &rest.Config{
URL: parseURL("http://hostname.foo:1234/"),
Connections: 5,
},
@ -299,7 +299,7 @@ var parseTests = []struct {
},
{
"b2:bucketname:/prefix", Location{Scheme: "b2",
Config: b2.Config{
Config: &b2.Config{
Bucket: "bucketname",
Prefix: "prefix",
Connections: 5,
@ -308,7 +308,7 @@ var parseTests = []struct {
},
{
"b2:bucketname", Location{Scheme: "b2",
Config: b2.Config{
Config: &b2.Config{
Bucket: "bucketname",
Prefix: "",
Connections: 5,

33
internal/backend/mem/mem_backend_test.go
View file

@ -15,19 +15,19 @@ type memConfig struct {
be restic.Backend
}
func newTestSuite() *test.Suite {
return &test.Suite{
func newTestSuite() *test.Suite[*memConfig] {
return &test.Suite[*memConfig]{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
return &memConfig{}, nil
NewConfig: func() (**memConfig, error) {
cfg := &memConfig{}
return &cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(cfg interface{}) (restic.Backend, error) {
c := cfg.(*memConfig)
if c.be != nil {
_, err := c.be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil && !c.be.IsNotExist(err) {
Create: func(cfg *memConfig) (restic.Backend, error) {
if cfg.be != nil {
_, err := cfg.be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil && !cfg.be.IsNotExist(err) {
return nil, err
}
@ -36,21 +36,20 @@ func newTestSuite() *test.Suite {
}
}
c.be = mem.New()
return c.be, nil
cfg.be = mem.New()
return cfg.be, nil
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(cfg interface{}) (restic.Backend, error) {
c := cfg.(*memConfig)
if c.be == nil {
c.be = mem.New()
Open: func(cfg *memConfig) (restic.Backend, error) {
if cfg.be == nil {
cfg.be = mem.New()
}
return c.be, nil
return cfg.be, nil
},
// CleanupFn removes data created during the tests.
Cleanup: func(cfg interface{}) error {
Cleanup: func(cfg *memConfig) error {
// no cleanup needed
return nil
},

14
internal/backend/rclone/backend_test.go
View file

@ -12,22 +12,21 @@ import (
rtest "github.com/restic/restic/internal/test"
)
func newTestSuite(t testing.TB) *test.Suite {
func newTestSuite(t testing.TB) *test.Suite[rclone.Config] {
dir := rtest.TempDir(t)
return &test.Suite{
return &test.Suite[rclone.Config]{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
NewConfig: func() (*rclone.Config, error) {
t.Logf("use backend at %v", dir)
cfg := rclone.NewConfig()
cfg.Remote = dir
return cfg, nil
return &cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
Create: func(cfg rclone.Config) (restic.Backend, error) {
t.Logf("Create()")
cfg := config.(rclone.Config)
be, err := rclone.Create(context.TODO(), cfg)
var e *exec.Error
if errors.As(err, &e) && e.Err == exec.ErrNotFound {
@ -38,9 +37,8 @@ func newTestSuite(t testing.TB) *test.Suite {
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
Open: func(cfg rclone.Config) (restic.Backend, error) {
t.Logf("Open()")
cfg := config.(rclone.Config)
return rclone.Open(cfg, nil)
},
}

4
internal/backend/rclone/config.go
View file

@ -34,7 +34,7 @@ func NewConfig() Config {
}
// ParseConfig parses the string s and extracts the remote server URL.
func ParseConfig(s string) (interface{}, error) {
func ParseConfig(s string) (*Config, error) {
if !strings.HasPrefix(s, "rclone:") {
return nil, errors.New("invalid rclone backend specification")
}
@ -42,5 +42,5 @@ func ParseConfig(s string) (interface{}, error) {
s = s[7:]
cfg := NewConfig()
cfg.Remote = s
return cfg, nil
return &cfg, nil
}

27
internal/backend/rclone/config_test.go
View file

@ -1,18 +1,15 @@
package rclone
import (
"reflect"
"testing"
"github.com/restic/restic/internal/backend/test"
)
func TestParseConfig(t *testing.T) {
var tests = []struct {
s string
cfg Config
}{
var configTests = []test.ConfigTestData[Config]{
{
"rclone:local:foo:/bar",
Config{
S: "rclone:local:foo:/bar",
Cfg: Config{
Remote: "local:foo:/bar",
Program: defaultConfig.Program,
Args: defaultConfig.Args,
@ -22,16 +19,6 @@ func TestParseConfig(t *testing.T) {
},
}
for _, test := range tests {
t.Run("", func(t *testing.T) {
cfg, err := ParseConfig(test.s)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(cfg, test.cfg) {
t.Fatalf("wrong config, want:\n %v\ngot:\n %v", test.cfg, cfg)
}
})
}
func TestParseConfig(t *testing.T) {
test.ParseConfigTester(t, ParseConfig, configTests)
}

4
internal/backend/rest/config.go
View file

@ -26,7 +26,7 @@ func NewConfig() Config {
}
// ParseConfig parses the string s and extracts the REST server URL.
func ParseConfig(s string) (interface{}, error) {
func ParseConfig(s string) (*Config, error) {
if !strings.HasPrefix(s, "rest:") {
return nil, errors.New("invalid REST backend specification")
}
@ -40,7 +40,7 @@ func ParseConfig(s string) (interface{}, error) {
cfg := NewConfig()
cfg.URL = u
return cfg, nil
return &cfg, nil
}
// StripPassword removes the password from the URL

30
internal/backend/rest/config_test.go
View file

@ -2,8 +2,9 @@ package rest
import (
"net/url"
"reflect"
"testing"
"github.com/restic/restic/internal/backend/test"
)
func parseURL(s string) *url.URL {
@ -15,20 +16,17 @@ func parseURL(s string) *url.URL {
return u
}
var configTests = []struct {
s string
cfg Config
}{
var configTests = []test.ConfigTestData[Config]{
{
s: "rest:http://localhost:1234",
cfg: Config{
S: "rest:http://localhost:1234",
Cfg: Config{
URL: parseURL("http://localhost:1234/"),
Connections: 5,
},
},
{
s: "rest:http://localhost:1234/",
cfg: Config{
S: "rest:http://localhost:1234/",
Cfg: Config{
URL: parseURL("http://localhost:1234/"),
Connections: 5,
},
@ -36,17 +34,5 @@ var configTests = []struct {
}
func TestParseConfig(t *testing.T) {
for _, test := range configTests {
t.Run("", func(t *testing.T) {
cfg, err := ParseConfig(test.s)
if err != nil {
t.Fatalf("%s failed: %v", test.s, err)
}
if !reflect.DeepEqual(cfg, test.cfg) {
t.Fatalf("\ninput: %s\n wrong config, want:\n %v\ngot:\n %v",
test.s, test.cfg, cfg)
}
})
}
test.ParseConfigTester(t, ParseConfig, configTests)
}

20
internal/backend/rest/rest_test.go
View file

@ -67,36 +67,34 @@ func runRESTServer(ctx context.Context, t testing.TB, dir string) (*url.URL, fun
return url, cleanup
}
func newTestSuite(_ context.Context, t testing.TB, url *url.URL, minimalData bool) *test.Suite {
func newTestSuite(_ context.Context, t testing.TB, url *url.URL, minimalData bool) *test.Suite[rest.Config] {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
return &test.Suite{
return &test.Suite[rest.Config]{
MinimalData: minimalData,
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
NewConfig: func() (*rest.Config, error) {
cfg := rest.NewConfig()
cfg.URL = url
return cfg, nil
return &cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(rest.Config)
Create: func(cfg rest.Config) (restic.Backend, error) {
return rest.Create(context.TODO(), cfg, tr)
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(rest.Config)
Open: func(cfg rest.Config) (restic.Backend, error) {
return rest.Open(cfg, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
Cleanup: func(cfg rest.Config) error {
return nil
},
}
@ -130,12 +128,10 @@ func TestBackendRESTExternalServer(t *testing.T) {
t.Fatal(err)
}
c := cfg.(rest.Config)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
newTestSuite(ctx, t, c.URL, true).RunTests(t)
newTestSuite(ctx, t, cfg.URL, true).RunTests(t)
}
func BenchmarkBackendREST(t *testing.B) {

33
internal/backend/s3/config.go
View file

@ -2,11 +2,13 @@ package s3
import (
"net/url"
"os"
"path"
"strings"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains all configuration necessary to connect to an s3 compatible
@ -44,7 +46,7 @@ func init() {
// supported configuration formats are s3://host/bucketname/prefix and
// s3:host/bucketname/prefix. The host can also be a valid s3 region
// name. If no prefix is given the prefix "restic" will be used.
func ParseConfig(s string) (interface{}, error) {
func ParseConfig(s string) (*Config, error) {
switch {
case strings.HasPrefix(s, "s3:http"):
// assume that a URL has been specified, parse it and
@ -75,7 +77,7 @@ func ParseConfig(s string) (interface{}, error) {
return createConfig(endpoint, bucket, prefix, false)
}
func createConfig(endpoint, bucket, prefix string, useHTTP bool) (interface{}, error) {
func createConfig(endpoint, bucket, prefix string, useHTTP bool) (*Config, error) {
if endpoint == "" {
return nil, errors.New("s3: invalid format, host/region or bucket name not found")
}
@ -89,5 +91,30 @@ func createConfig(endpoint, bucket, prefix string, useHTTP bool) (interface{}, e
cfg.UseHTTP = useHTTP
cfg.Bucket = bucket
cfg.Prefix = prefix
return cfg, nil
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) error {
if cfg.KeyID == "" {
cfg.KeyID = os.Getenv(prefix + "AWS_ACCESS_KEY_ID")
}
if cfg.Secret.String() == "" {
cfg.Secret = options.NewSecretString(os.Getenv(prefix + "AWS_SECRET_ACCESS_KEY"))
}
if cfg.KeyID == "" && cfg.Secret.String() != "" {
return errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty")
} else if cfg.KeyID != "" && cfg.Secret.String() == "" {
return errors.Fatalf("unable to open S3 backend: Secret ($AWS_SECRET_ACCESS_KEY) is empty")
}
if cfg.Region == "" {
cfg.Region = os.Getenv(prefix + "AWS_DEFAULT_REGION")
}
return nil
}

49
internal/backend/s3/config_test.go
View file

@ -3,94 +3,93 @@ package s3
import (
"strings"
"testing"
"github.com/restic/restic/internal/backend/test"
)
var configTests = []struct {
s string
cfg Config
}{
{"s3://eu-central-1/bucketname", Config{
var configTests = []test.ConfigTestData[Config]{
{S: "s3://eu-central-1/bucketname", Cfg: Config{
Endpoint: "eu-central-1",
Bucket: "bucketname",
Prefix: "",
Connections: 5,
}},
{"s3://eu-central-1/bucketname/", Config{
{S: "s3://eu-central-1/bucketname/", Cfg: Config{
Endpoint: "eu-central-1",
Bucket: "bucketname",
Prefix: "",
Connections: 5,
}},
{"s3://eu-central-1/bucketname/prefix/directory", Config{
{S: "s3://eu-central-1/bucketname/prefix/directory", Cfg: Config{
Endpoint: "eu-central-1",
Bucket: "bucketname",
Prefix: "prefix/directory",
Connections: 5,
}},
{"s3://eu-central-1/bucketname/prefix/directory/", Config{
{S: "s3://eu-central-1/bucketname/prefix/directory/", Cfg: Config{
Endpoint: "eu-central-1",
Bucket: "bucketname",
Prefix: "prefix/directory",
Connections: 5,
}},
{"s3:eu-central-1/foobar", Config{
{S: "s3:eu-central-1/foobar", Cfg: Config{
Endpoint: "eu-central-1",
Bucket: "foobar",
Prefix: "",
Connections: 5,
}},
{"s3:eu-central-1/foobar/", Config{
{S: "s3:eu-central-1/foobar/", Cfg: Config{
Endpoint: "eu-central-1",
Bucket: "foobar",
Prefix: "",
Connections: 5,
}},
{"s3:eu-central-1/foobar/prefix/directory", Config{
{S: "s3:eu-central-1/foobar/prefix/directory", Cfg: Config{
Endpoint: "eu-central-1",
Bucket: "foobar",
Prefix: "prefix/directory",
Connections: 5,
}},
{"s3:eu-central-1/foobar/prefix/directory/", Config{
{S: "s3:eu-central-1/foobar/prefix/directory/", Cfg: Config{
Endpoint: "eu-central-1",
Bucket: "foobar",
Prefix: "prefix/directory",
Connections: 5,
}},
{"s3:https://hostname:9999/foobar", Config{
{S: "s3:https://hostname:9999/foobar", Cfg: Config{
Endpoint: "hostname:9999",
Bucket: "foobar",
Prefix: "",
Connections: 5,
}},
{"s3:https://hostname:9999/foobar/", Config{
{S: "s3:https://hostname:9999/foobar/", Cfg: Config{
Endpoint: "hostname:9999",
Bucket: "foobar",
Prefix: "",
Connections: 5,
}},
{"s3:http://hostname:9999/foobar", Config{
{S: "s3:http://hostname:9999/foobar", Cfg: Config{
Endpoint: "hostname:9999",
Bucket: "foobar",
Prefix: "",
UseHTTP: true,
Connections: 5,
}},
{"s3:http://hostname:9999/foobar/", Config{
{S: "s3:http://hostname:9999/foobar/", Cfg: Config{
Endpoint: "hostname:9999",
Bucket: "foobar",
Prefix: "",
UseHTTP: true,
Connections: 5,
}},
{"s3:http://hostname:9999/bucket/prefix/directory", Config{
{S: "s3:http://hostname:9999/bucket/prefix/directory", Cfg: Config{
Endpoint: "hostname:9999",
Bucket: "bucket",
Prefix: "prefix/directory",
UseHTTP: true,
Connections: 5,
}},
{"s3:http://hostname:9999/bucket/prefix/directory/", Config{
{S: "s3:http://hostname:9999/bucket/prefix/directory/", Cfg: Config{
Endpoint: "hostname:9999",
Bucket: "bucket",
Prefix: "prefix/directory",
@ -100,19 +99,7 @@ var configTests = []struct {
}
func TestParseConfig(t *testing.T) {
for i, test := range configTests {
cfg, err := ParseConfig(test.s)
if err != nil {
t.Errorf("test %d:%s failed: %v", i, test.s, err)
continue
}
if cfg != test.cfg {
t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v",
i, test.s, test.cfg, cfg)
continue
}
}
test.ParseConfigTester(t, ParseConfig, configTests)
}
func TestParseError(t *testing.T) {

38
internal/backend/s3/s3_test.go
View file

@ -120,15 +120,15 @@ func createS3(t testing.TB, cfg MinioTestConfig, tr http.RoundTripper) (be resti
return be, err
}
func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite[MinioTestConfig] {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
return &test.Suite{
return &test.Suite[MinioTestConfig]{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
NewConfig: func() (*MinioTestConfig, error) {
cfg := MinioTestConfig{}
cfg.tempdir = rtest.TempDir(t)
@ -142,13 +142,11 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
cfg.Config.UseHTTP = true
cfg.Config.KeyID = key
cfg.Config.Secret = options.NewSecretString(secret)
return cfg, nil
return &cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(MinioTestConfig)
Create: func(cfg MinioTestConfig) (restic.Backend, error) {
be, err := createS3(t, cfg, tr)
if err != nil {
return nil, err
@ -167,14 +165,12 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(MinioTestConfig)
Open: func(cfg MinioTestConfig) (restic.Backend, error) {
return s3.Open(ctx, cfg.Config, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
cfg := config.(MinioTestConfig)
Cleanup: func(cfg MinioTestConfig) error {
if cfg.stopServer != nil {
cfg.stopServer()
}
@ -217,24 +213,23 @@ func BenchmarkBackendMinio(t *testing.B) {
newMinioTestSuite(ctx, t).RunBenchmarks(t)
}
func newS3TestSuite(t testing.TB) *test.Suite {
func newS3TestSuite(t testing.TB) *test.Suite[s3.Config] {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
return &test.Suite{
return &test.Suite[s3.Config]{
// do not use excessive data
MinimalData: true,
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
s3cfg, err := s3.ParseConfig(os.Getenv("RESTIC_TEST_S3_REPOSITORY"))
NewConfig: func() (*s3.Config, error) {
cfg, err := s3.ParseConfig(os.Getenv("RESTIC_TEST_S3_REPOSITORY"))
if err != nil {
return nil, err
}
cfg := s3cfg.(s3.Config)
cfg.KeyID = os.Getenv("RESTIC_TEST_S3_KEY")
cfg.Secret = options.NewSecretString(os.Getenv("RESTIC_TEST_S3_SECRET"))
cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
@ -242,9 +237,7 @@ func newS3TestSuite(t testing.TB) *test.Suite {
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(s3.Config)
Create: func(cfg s3.Config) (restic.Backend, error) {
be, err := s3.Create(context.TODO(), cfg, tr)
if err != nil {
return nil, err
@ -263,15 +256,12 @@ func newS3TestSuite(t testing.TB) *test.Suite {
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(s3.Config)
Open: func(cfg s3.Config) (restic.Backend, error) {
return s3.Open(context.TODO(), cfg, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
cfg := config.(s3.Config)
Cleanup: func(cfg s3.Config) error {
be, err := s3.Open(context.TODO(), cfg, tr)
if err != nil {
return err

4
internal/backend/sftp/config.go
View file

@ -35,7 +35,7 @@ func init() {
// and sftp:user@host:directory. The directory will be path Cleaned and can
// be an absolute path if it starts with a '/' (e.g.
// sftp://user@host//absolute and sftp:user@host:/absolute).
func ParseConfig(s string) (interface{}, error) {
func ParseConfig(s string) (*Config, error) {
var user, host, port, dir string
switch {
case strings.HasPrefix(s, "sftp://"):
@ -89,5 +89,5 @@ func ParseConfig(s string) (interface{}, error) {
cfg.Port = port
cfg.Path = p
return cfg, nil
return &cfg, nil
}

81
internal/backend/sftp/config_test.go
View file

@ -2,94 +2,81 @@ package sftp
import (
"testing"
"github.com/restic/restic/internal/backend/test"
)
var configTests = []struct {
in string
cfg Config
}{
var configTests = []test.ConfigTestData[Config]{
// first form, user specified sftp://user@host/dir
{
"sftp://user@host/dir/subdir",
Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5},
S: "sftp://user@host/dir/subdir",
Cfg: Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5},
},
{
"sftp://host/dir/subdir",
Config{Host: "host", Path: "dir/subdir", Connections: 5},
S: "sftp://host/dir/subdir",
Cfg: Config{Host: "host", Path: "dir/subdir", Connections: 5},
},
{
"sftp://host//dir/subdir",
Config{Host: "host", Path: "/dir/subdir", Connections: 5},
S: "sftp://host//dir/subdir",
Cfg: Config{Host: "host", Path: "/dir/subdir", Connections: 5},
},
{
"sftp://host:10022//dir/subdir",
Config{Host: "host", Port: "10022", Path: "/dir/subdir", Connections: 5},
S: "sftp://host:10022//dir/subdir",
Cfg: Config{Host: "host", Port: "10022", Path: "/dir/subdir", Connections: 5},
},
{
"sftp://user@host:10022//dir/subdir",
Config{User: "user", Host: "host", Port: "10022", Path: "/dir/subdir", Connections: 5},
S: "sftp://user@host:10022//dir/subdir",
Cfg: Config{User: "user", Host: "host", Port: "10022", Path: "/dir/subdir", Connections: 5},
},
{
"sftp://user@host/dir/subdir/../other",
Config{User: "user", Host: "host", Path: "dir/other", Connections: 5},
S: "sftp://user@host/dir/subdir/../other",
Cfg: Config{User: "user", Host: "host", Path: "dir/other", Connections: 5},
},
{
"sftp://user@host/dir///subdir",
Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5},
S: "sftp://user@host/dir///subdir",
Cfg: Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5},
},
// IPv6 address.
{
"sftp://user@[::1]/dir",
Config{User: "user", Host: "::1", Path: "dir", Connections: 5},
S: "sftp://user@[::1]/dir",
Cfg: Config{User: "user", Host: "::1", Path: "dir", Connections: 5},
},
// IPv6 address with port.
{
"sftp://user@[::1]:22/dir",
Config{User: "user", Host: "::1", Port: "22", Path: "dir", Connections: 5},
S: "sftp://user@[::1]:22/dir",
Cfg: Config{User: "user", Host: "::1", Port: "22", Path: "dir", Connections: 5},
},
// second form, user specified sftp:user@host:/dir
{
"sftp:user@host:/dir/subdir",
Config{User: "user", Host: "host", Path: "/dir/subdir", Connections: 5},
S: "sftp:user@host:/dir/subdir",
Cfg: Config{User: "user", Host: "host", Path: "/dir/subdir", Connections: 5},
},
{
"sftp:user@domain@host:/dir/subdir",
Config{User: "user@domain", Host: "host", Path: "/dir/subdir", Connections: 5},
S: "sftp:user@domain@host:/dir/subdir",
Cfg: Config{User: "user@domain", Host: "host", Path: "/dir/subdir", Connections: 5},
},
{
"sftp:host:../dir/subdir",
Config{Host: "host", Path: "../dir/subdir", Connections: 5},
S: "sftp:host:../dir/subdir",
Cfg: Config{Host: "host", Path: "../dir/subdir", Connections: 5},
},
{
"sftp:user@host:dir/subdir:suffix",
Config{User: "user", Host: "host", Path: "dir/subdir:suffix", Connections: 5},
S: "sftp:user@host:dir/subdir:suffix",
Cfg: Config{User: "user", Host: "host", Path: "dir/subdir:suffix", Connections: 5},
},
{
"sftp:user@host:dir/subdir/../other",
Config{User: "user", Host: "host", Path: "dir/other", Connections: 5},
S: "sftp:user@host:dir/subdir/../other",
Cfg: Config{User: "user", Host: "host", Path: "dir/other", Connections: 5},
},
{
"sftp:user@host:dir///subdir",
Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5},
S: "sftp:user@host:dir///subdir",
Cfg: Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5},
},
}
func TestParseConfig(t *testing.T) {
for i, test := range configTests {
cfg, err := ParseConfig(test.in)
if err != nil {
t.Errorf("test %d:%s failed: %v", i, test.in, err)
continue
}
if cfg != test.cfg {
t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v",
i, test.in, test.cfg, cfg)
continue
}
}
test.ParseConfigTester(t, ParseConfig, configTests)
}
var configTestsInvalid = []string{

17
internal/backend/sftp/sftp_test.go
View file

@ -29,10 +29,10 @@ func findSFTPServerBinary() string {
var sftpServer = findSFTPServerBinary()
func newTestSuite(t testing.TB) *test.Suite {
return &test.Suite{
func newTestSuite(t testing.TB) *test.Suite[sftp.Config] {
return &test.Suite[sftp.Config]{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
NewConfig: func() (*sftp.Config, error) {
dir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-sftp-")
if err != nil {
t.Fatal(err)
@ -40,7 +40,7 @@ func newTestSuite(t testing.TB) *test.Suite {
t.Logf("create new backend at %v", dir)
cfg := sftp.Config{
cfg := &sftp.Config{
Path: dir,
Command: fmt.Sprintf("%q -e", sftpServer),
Connections: 5,
@ -49,20 +49,17 @@ func newTestSuite(t testing.TB) *test.Suite {
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(sftp.Config)
Create: func(cfg sftp.Config) (restic.Backend, error) {
return sftp.Create(context.TODO(), cfg)
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(sftp.Config)
Open: func(cfg sftp.Config) (restic.Backend, error) {
return sftp.Open(context.TODO(), cfg)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
cfg := config.(sftp.Config)
Cleanup: func(cfg sftp.Config) error {
if !rtest.TestCleanupTempDirs {
t.Logf("leaving test backend dir at %v", cfg.Path)
}

29
internal/backend/smb/config.go
View file

@ -2,6 +2,7 @@ package smb
import (
"net/url"
"os"
"path"
"strconv"
"strings"
@ -9,6 +10,7 @@ import (
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains all configuration necessary to connect to an SMB server
@ -54,7 +56,7 @@ func init() {
// ParseConfig parses the string s and extracts the s3 config. The
// supported configuration format is smb://[user@]host[:port]/sharename/directory.
// User and port are optional. Default port is 445.
func ParseConfig(s string) (interface{}, error) {
func ParseConfig(s string) (*Config, error) {
var repo string
switch {
case strings.HasPrefix(s, "smb://"):
@ -106,7 +108,7 @@ func ParseConfig(s string) (interface{}, error) {
return createConfig(user, host, portNum, sharename, directory)
}
func createConfig(user string, host string, port int, sharename, directory string) (interface{}, error) {
func createConfig(user string, host string, port int, sharename, directory string) (*Config, error) {
if host == "" {
return nil, errors.New("smb: invalid format, Host not found")
}
@ -121,5 +123,26 @@ func createConfig(user string, host string, port int, sharename, directory strin
cfg.Port = port
cfg.ShareName = sharename
cfg.Path = directory
return cfg, nil
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) error {
if cfg.User == "" {
cfg.User = os.Getenv("RESTIC_SMB_USER")
}
if cfg.Password.String() == "" {
cfg.Password = options.NewSecretString(os.Getenv("RESTIC_SMB_PASSWORD"))
}
if cfg.Domain == "" {
cfg.Domain = os.Getenv("RESTIC_SMB_DOMAIN")
}
if cfg.Domain == "" {
cfg.Domain = DefaultDomain
}
return nil
}

36
internal/backend/smb/config_test.go
View file

@ -3,13 +3,13 @@ package smb
import (
"strings"
"testing"
"github.com/restic/restic/internal/backend/test"
)
var configTests = []struct {
s string
cfg Config
}{
{"smb://user@host/sharename/directory", Config{
var configTests = []test.ConfigTestData[Config]{
{S: "smb://user@host/sharename/directory",
Cfg: Config{
Host: "host",
Port: DefaultSmbPort,
User: "user",
@ -19,7 +19,8 @@ var configTests = []struct {
Connections: DefaultConnections,
IdleTimeout: DefaultIdleTimeout,
}},
{"smb://user@host:456/sharename/directory", Config{
{S: "smb://user@host:456/sharename/directory",
Cfg: Config{
Host: "host",
Port: 456,
User: "user",
@ -29,7 +30,8 @@ var configTests = []struct {
Connections: DefaultConnections,
IdleTimeout: DefaultIdleTimeout,
}},
{"smb://host/sharename/directory", Config{
{S: "smb://host/sharename/directory",
Cfg: Config{
Host: "host",
Port: DefaultSmbPort,
Domain: DefaultDomain,
@ -38,7 +40,8 @@ var configTests = []struct {
Connections: DefaultConnections,
IdleTimeout: DefaultIdleTimeout,
}},
{"smb://host:446/sharename/directory", Config{
{S: "smb://host:446/sharename/directory",
Cfg: Config{
Host: "host",
Port: 446,
Domain: DefaultDomain,
@ -47,7 +50,8 @@ var configTests = []struct {
Connections: DefaultConnections,
IdleTimeout: DefaultIdleTimeout,
}},
{"smb:user@host:466/sharename/directory", Config{
{S: "smb:user@host:466/sharename/directory",
Cfg: Config{
Host: "host",
Port: 466,
User: "user",
@ -60,19 +64,7 @@ var configTests = []struct {
}
func TestParseConfig(t *testing.T) {
for i, test := range configTests {
cfg, err := ParseConfig(test.s)
if err != nil {
t.Errorf("test %d:%s failed: %v", i, test.s, err)
continue
}
if cfg != test.cfg {
t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v",
i, test.s, test.cfg, cfg)
continue
}
}
test.ParseConfigTester(t, ParseConfig, configTests)
}
func TestParseError(t *testing.T) {

17
internal/backend/smb/smb_test.go
View file

@ -13,12 +13,12 @@ import (
rtest "github.com/restic/restic/internal/test"
)
func newTestSuite(t testing.TB) *test.Suite {
return &test.Suite{
func newTestSuite(t testing.TB) *test.Suite[smb.Config] {
return &test.Suite[smb.Config]{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
NewConfig: func() (*smb.Config, error) {
cfg := smb.NewConfig()
cfg := &smb.Config{}
cfg.Host = "127.0.0.1"
cfg.User = "smbuser"
cfg.ShareName = cfg.User
@ -38,20 +38,17 @@ func newTestSuite(t testing.TB) *test.Suite {
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(smb.Config)
Create: func(cfg smb.Config) (restic.Backend, error) {
return smb.Create(context.TODO(), cfg)
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(smb.Config)
Open: func(cfg smb.Config) (restic.Backend, error) {
return smb.Open(context.TODO(), cfg)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
cfg := config.(smb.Config)
Cleanup: func(cfg smb.Config) error {
if !rtest.TestCleanupTempDirs {
t.Logf("leaving test backend dir at %v", cfg.Path)
}

54
internal/backend/swift/config.go
View file

@ -6,6 +6,7 @@ import (
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains basic configuration needed to specify swift location for a swift server
@ -50,7 +51,7 @@ func NewConfig() Config {
}
// ParseConfig parses the string s and extract swift's container name and prefix.
func ParseConfig(s string) (interface{}, error) {
func ParseConfig(s string) (*Config, error) {
if !strings.HasPrefix(s, "swift:") {
return nil, errors.New("invalid URL, expected: swift:container-name:/[prefix]")
}
@ -70,48 +71,49 @@ func ParseConfig(s string) (interface{}, error) {
cfg.Container = container
cfg.Prefix = prefix
return cfg, nil
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func ApplyEnvironment(prefix string, cfg interface{}) error {
c := cfg.(*Config)
func (cfg *Config) ApplyEnvironment(prefix string) error {
for _, val := range []struct {
s *string
env string
}{
// v2/v3 specific
{&c.UserName, prefix + "OS_USERNAME"},
{&c.APIKey, prefix + "OS_PASSWORD"},
{&c.Region, prefix + "OS_REGION_NAME"},
{&c.AuthURL, prefix + "OS_AUTH_URL"},
{&cfg.UserName, prefix + "OS_USERNAME"},
{&cfg.APIKey, prefix + "OS_PASSWORD"},
{&cfg.Region, prefix + "OS_REGION_NAME"},
{&cfg.AuthURL, prefix + "OS_AUTH_URL"},
// v3 specific
{&c.UserID, prefix + "OS_USER_ID"},
{&c.Domain, prefix + "OS_USER_DOMAIN_NAME"},
{&c.DomainID, prefix + "OS_USER_DOMAIN_ID"},
{&c.Tenant, prefix + "OS_PROJECT_NAME"},
{&c.TenantDomain, prefix + "OS_PROJECT_DOMAIN_NAME"},
{&c.TenantDomainID, prefix + "OS_PROJECT_DOMAIN_ID"},
{&c.TrustID, prefix + "OS_TRUST_ID"},
{&cfg.UserID, prefix + "OS_USER_ID"},
{&cfg.Domain, prefix + "OS_USER_DOMAIN_NAME"},
{&cfg.DomainID, prefix + "OS_USER_DOMAIN_ID"},
{&cfg.Tenant, prefix + "OS_PROJECT_NAME"},
{&cfg.TenantDomain, prefix + "OS_PROJECT_DOMAIN_NAME"},
{&cfg.TenantDomainID, prefix + "OS_PROJECT_DOMAIN_ID"},
{&cfg.TrustID, prefix + "OS_TRUST_ID"},
// v2 specific
{&c.TenantID, prefix + "OS_TENANT_ID"},
{&c.Tenant, prefix + "OS_TENANT_NAME"},
{&cfg.TenantID, prefix + "OS_TENANT_ID"},
{&cfg.Tenant, prefix + "OS_TENANT_NAME"},
// v1 specific
{&c.AuthURL, prefix + "ST_AUTH"},
{&c.UserName, prefix + "ST_USER"},
{&c.APIKey, prefix + "ST_KEY"},
{&cfg.AuthURL, prefix + "ST_AUTH"},
{&cfg.UserName, prefix + "ST_USER"},
{&cfg.APIKey, prefix + "ST_KEY"},
// Application Credential auth
{&c.ApplicationCredentialID, prefix + "OS_APPLICATION_CREDENTIAL_ID"},
{&c.ApplicationCredentialName, prefix + "OS_APPLICATION_CREDENTIAL_NAME"},
{&cfg.ApplicationCredentialID, prefix + "OS_APPLICATION_CREDENTIAL_ID"},
{&cfg.ApplicationCredentialName, prefix + "OS_APPLICATION_CREDENTIAL_NAME"},
// Manual authentication
{&c.StorageURL, prefix + "OS_STORAGE_URL"},
{&cfg.StorageURL, prefix + "OS_STORAGE_URL"},
{&c.DefaultContainerPolicy, prefix + "SWIFT_DEFAULT_CONTAINER_POLICY"},
{&cfg.DefaultContainerPolicy, prefix + "SWIFT_DEFAULT_CONTAINER_POLICY"},
} {
if *val.s == "" {
*val.s = os.Getenv(val.env)
@ -121,8 +123,8 @@ func ApplyEnvironment(prefix string, cfg interface{}) error {
s *options.SecretString
env string
}{
{&c.ApplicationCredentialSecret, prefix + "OS_APPLICATION_CREDENTIAL_SECRET"},
{&c.AuthToken, prefix + "OS_AUTH_TOKEN"},
{&cfg.ApplicationCredentialSecret, prefix + "OS_APPLICATION_CREDENTIAL_SECRET"},
{&cfg.AuthToken, prefix + "OS_AUTH_TOKEN"},
} {
if val.s.String() == "" {
*val.s = options.NewSecretString(os.Getenv(val.env))

42
internal/backend/swift/config_test.go
View file

@ -1,29 +1,30 @@
package swift
import "testing"
import (
"testing"
var configTests = []struct {
s string
cfg Config
}{
"github.com/restic/restic/internal/backend/test"
)
var configTests = []test.ConfigTestData[Config]{
{
"swift:cnt1:/",
Config{
S: "swift:cnt1:/",
Cfg: Config{
Container: "cnt1",
Prefix: "",
Connections: 5,
},
},
{
"swift:cnt2:/prefix",
Config{Container: "cnt2",
S: "swift:cnt2:/prefix",
Cfg: Config{Container: "cnt2",
Prefix: "prefix",
Connections: 5,
},
},
{
"swift:cnt3:/prefix/longer",
Config{Container: "cnt3",
S: "swift:cnt3:/prefix/longer",
Cfg: Config{Container: "cnt3",
Prefix: "prefix/longer",
Connections: 5,
},
@ -31,24 +32,7 @@ var configTests = []struct {
}
func TestParseConfig(t *testing.T) {
for _, test := range configTests {
t.Run("", func(t *testing.T) {
v, err := ParseConfig(test.s)
if err != nil {
t.Fatalf("parsing %q failed: %v", test.s, err)
}
cfg, ok := v.(Config)
if !ok {
t.Fatalf("wrong type returned, want Config, got %T", cfg)
}
if cfg != test.cfg {
t.Fatalf("wrong output for %q, want:\n %#v\ngot:\n %#v",
test.s, test.cfg, cfg)
}
})
}
test.ParseConfigTester(t, ParseConfig, configTests)
}
var configTestsInvalid = []string{

22
internal/backend/swift/swift_test.go
View file

@ -15,13 +15,13 @@ import (
rtest "github.com/restic/restic/internal/test"
)
func newSwiftTestSuite(t testing.TB) *test.Suite {
func newSwiftTestSuite(t testing.TB) *test.Suite[swift.Config] {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
return &test.Suite{
return &test.Suite[swift.Config]{
// do not use excessive data
MinimalData: true,
@ -42,14 +42,13 @@ func newSwiftTestSuite(t testing.TB) *test.Suite {
},
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
swiftcfg, err := swift.ParseConfig(os.Getenv("RESTIC_TEST_SWIFT"))
NewConfig: func() (*swift.Config, error) {
cfg, err := swift.ParseConfig(os.Getenv("RESTIC_TEST_SWIFT"))
if err != nil {
return nil, err
}
cfg := swiftcfg.(swift.Config)
if err = swift.ApplyEnvironment("RESTIC_TEST_", &cfg); err != nil {
if err = cfg.ApplyEnvironment("RESTIC_TEST_"); err != nil {
return nil, err
}
cfg.Prefix += fmt.Sprintf("/test-%d", time.Now().UnixNano())
@ -58,9 +57,7 @@ func newSwiftTestSuite(t testing.TB) *test.Suite {
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(swift.Config)
Create: func(cfg swift.Config) (restic.Backend, error) {
be, err := swift.Open(context.TODO(), cfg, tr)
if err != nil {
return nil, err
@ -79,15 +76,12 @@ func newSwiftTestSuite(t testing.TB) *test.Suite {
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(swift.Config)
Open: func(cfg swift.Config) (restic.Backend, error) {
return swift.Open(context.TODO(), cfg, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
cfg := config.(swift.Config)
Cleanup: func(cfg swift.Config) error {
be, err := swift.Open(context.TODO(), cfg, tr)
if err != nil {
return err

8
internal/backend/test/benchmarks.go
View file

@ -29,7 +29,7 @@ func remove(t testing.TB, be restic.Backend, h restic.Handle) {
// BenchmarkLoadFile benchmarks the Load() method of a backend by
// loading a complete file.
func (s *Suite) BenchmarkLoadFile(t *testing.B) {
func (s *Suite[C]) BenchmarkLoadFile(t *testing.B) {
be := s.open(t)
defer s.close(t, be)
@ -64,7 +64,7 @@ func (s *Suite) BenchmarkLoadFile(t *testing.B) {
// BenchmarkLoadPartialFile benchmarks the Load() method of a backend by
// loading the remainder of a file starting at a given offset.
func (s *Suite) BenchmarkLoadPartialFile(t *testing.B) {
func (s *Suite[C]) BenchmarkLoadPartialFile(t *testing.B) {
be := s.open(t)
defer s.close(t, be)
@ -101,7 +101,7 @@ func (s *Suite) BenchmarkLoadPartialFile(t *testing.B) {
// BenchmarkLoadPartialFileOffset benchmarks the Load() method of a
// backend by loading a number of bytes of a file starting at a given offset.
func (s *Suite) BenchmarkLoadPartialFileOffset(t *testing.B) {
func (s *Suite[C]) BenchmarkLoadPartialFileOffset(t *testing.B) {
be := s.open(t)
defer s.close(t, be)
@ -139,7 +139,7 @@ func (s *Suite) BenchmarkLoadPartialFileOffset(t *testing.B) {
}
// BenchmarkSave benchmarks the Save() method of a backend.
func (s *Suite) BenchmarkSave(t *testing.B) {
func (s *Suite[C]) BenchmarkSave(t *testing.B) {
be := s.open(t)
defer s.close(t, be)

28
internal/backend/test/config.go Normal file
View file

@ -0,0 +1,28 @@
package test
import (
"fmt"
"reflect"
"testing"
)
type ConfigTestData[C comparable] struct {
S string
Cfg C
}
func ParseConfigTester[C comparable](t *testing.T, parser func(s string) (*C, error), tests []ConfigTestData[C]) {
for i, test := range tests {
t.Run(fmt.Sprint(i), func(t *testing.T) {
cfg, err := parser(test.S)
if err != nil {
t.Fatalf("%s failed: %v", test.S, err)
}
if !reflect.DeepEqual(*cfg, test.Cfg) {
t.Fatalf("input: %s\n wrong config, want:\n %#v\ngot:\n %#v",
test.S, test.Cfg, *cfg)
}
})
}
}

34
internal/backend/test/suite.go
View file

@ -11,21 +11,21 @@ import (
)
// Suite implements a test suite for restic backends.
type Suite struct {
type Suite[C any] struct {
// Config should be used to configure the backend.
Config interface{}
Config *C
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig func() (interface{}, error)
NewConfig func() (*C, error)
// CreateFn is a function that creates a temporary repository for the tests.
Create func(cfg interface{}) (restic.Backend, error)
Create func(cfg C) (restic.Backend, error)
// OpenFn is a function that opens a previously created temporary repository.
Open func(cfg interface{}) (restic.Backend, error)
Open func(cfg C) (restic.Backend, error)
// CleanupFn removes data created during the tests.
Cleanup func(cfg interface{}) error
Cleanup func(cfg C) error
// MinimalData instructs the tests to not use excessive data.
MinimalData bool
@ -40,7 +40,7 @@ type Suite struct {
}
// RunTests executes all defined tests as subtests of t.
func (s *Suite) RunTests(t *testing.T) {
func (s *Suite[C]) RunTests(t *testing.T) {
var err error
s.Config, err = s.NewConfig()
if err != nil {
@ -61,7 +61,7 @@ func (s *Suite) RunTests(t *testing.T) {
}
if s.Cleanup != nil {
if err = s.Cleanup(s.Config); err != nil {
if err = s.Cleanup(*s.Config); err != nil {
t.Fatal(err)
}
}
@ -72,7 +72,7 @@ type testFunction struct {
Fn func(*testing.T)
}
func (s *Suite) testFuncs(t testing.TB) (funcs []testFunction) {
func (s *Suite[C]) testFuncs(t testing.TB) (funcs []testFunction) {
tpe := reflect.TypeOf(s)
v := reflect.ValueOf(s)
@ -107,7 +107,7 @@ type benchmarkFunction struct {
Fn func(*testing.B)
}
func (s *Suite) benchmarkFuncs(t testing.TB) (funcs []benchmarkFunction) {
func (s *Suite[C]) benchmarkFuncs(t testing.TB) (funcs []benchmarkFunction) {
tpe := reflect.TypeOf(s)
v := reflect.ValueOf(s)
@ -138,7 +138,7 @@ func (s *Suite) benchmarkFuncs(t testing.TB) (funcs []benchmarkFunction) {
}
// RunBenchmarks executes all defined benchmarks as subtests of b.
func (s *Suite) RunBenchmarks(b *testing.B) {
func (s *Suite[C]) RunBenchmarks(b *testing.B) {
var err error
s.Config, err = s.NewConfig()
if err != nil {
@ -158,28 +158,28 @@ func (s *Suite) RunBenchmarks(b *testing.B) {
return
}
if err = s.Cleanup(s.Config); err != nil {
if err = s.Cleanup(*s.Config); err != nil {
b.Fatal(err)
}
}
func (s *Suite) create(t testing.TB) restic.Backend {
be, err := s.Create(s.Config)
func (s *Suite[C]) create(t testing.TB) restic.Backend {
be, err := s.Create(*s.Config)
if err != nil {
t.Fatal(err)
}
return be
}
func (s *Suite) open(t testing.TB) restic.Backend {
be, err := s.Open(s.Config)
func (s *Suite[C]) open(t testing.TB) restic.Backend {
be, err := s.Open(*s.Config)
if err != nil {
t.Fatal(err)
}
return be
}
func (s *Suite) close(t testing.TB, be restic.Backend) {
func (s *Suite[C]) close(t testing.TB, be restic.Backend) {
err := be.Close()
if err != nil {
t.Fatal(err)

34
internal/backend/test/tests.go
View file

@ -38,7 +38,7 @@ func beTest(ctx context.Context, be restic.Backend, h restic.Handle) (bool, erro
// TestCreateWithConfig tests that creating a backend in a location which already
// has a config file fails.
func (s *Suite) TestCreateWithConfig(t *testing.T) {
func (s *Suite[C]) TestCreateWithConfig(t *testing.T) {
b := s.open(t)
defer s.close(t, b)
@ -57,7 +57,7 @@ func (s *Suite) TestCreateWithConfig(t *testing.T) {
store(t, b, restic.ConfigFile, []byte("test config"))
// now create the backend again, this must fail
_, err = s.Create(s.Config)
_, err = s.Create(*s.Config)
if err == nil {
t.Fatalf("expected error not found for creating a backend with an existing config file")
}
@ -70,7 +70,7 @@ func (s *Suite) TestCreateWithConfig(t *testing.T) {
}
// TestLocation tests that a location string is returned.
func (s *Suite) TestLocation(t *testing.T) {
func (s *Suite[C]) TestLocation(t *testing.T) {
b := s.open(t)
defer s.close(t, b)
@ -81,7 +81,7 @@ func (s *Suite) TestLocation(t *testing.T) {
}
// TestConfig saves and loads a config from the backend.
func (s *Suite) TestConfig(t *testing.T) {
func (s *Suite[C]) TestConfig(t *testing.T) {
b := s.open(t)
defer s.close(t, b)
@ -118,7 +118,7 @@ func (s *Suite) TestConfig(t *testing.T) {
}
// TestLoad tests the backend's Load function.
func (s *Suite) TestLoad(t *testing.T) {
func (s *Suite[C]) TestLoad(t *testing.T) {
seedRand(t)
b := s.open(t)
@ -222,8 +222,12 @@ func (s *Suite) TestLoad(t *testing.T) {
test.OK(t, b.Remove(context.TODO(), handle))
}
type setter interface {
SetListMaxItems(int)
}
// TestList makes sure that the backend implements List() pagination correctly.
func (s *Suite) TestList(t *testing.T) {
func (s *Suite[C]) TestList(t *testing.T) {
seedRand(t)
numTestFiles := rand.Intn(20) + 20
@ -269,10 +273,6 @@ func (s *Suite) TestList(t *testing.T) {
t.Run(fmt.Sprintf("max-%v", test.maxItems), func(t *testing.T) {
list2 := make(map[restic.ID]int64)
type setter interface {
SetListMaxItems(int)
}
if s, ok := b.(setter); ok {
t.Logf("setting max list items to %d", test.maxItems)
s.SetListMaxItems(test.maxItems)
@ -326,7 +326,7 @@ func (s *Suite) TestList(t *testing.T) {
}
// TestListCancel tests that the context is respected and the error is returned by List.
func (s *Suite) TestListCancel(t *testing.T) {
func (s *Suite[C]) TestListCancel(t *testing.T) {
seedRand(t)
numTestFiles := 5
@ -466,7 +466,7 @@ func (ec errorCloser) Rewind() error {
}
// TestSave tests saving data in the backend.
func (s *Suite) TestSave(t *testing.T) {
func (s *Suite[C]) TestSave(t *testing.T) {
seedRand(t)
b := s.open(t)
@ -582,7 +582,7 @@ func (r *incompleteByteReader) Length() int64 {
}
// TestSaveError tests saving data in the backend.
func (s *Suite) TestSaveError(t *testing.T) {
func (s *Suite[C]) TestSaveError(t *testing.T) {
seedRand(t)
b := s.open(t)
@ -621,7 +621,7 @@ func (b *wrongByteReader) Hash() []byte {
}
// TestSaveWrongHash tests that uploads with a wrong hash fail
func (s *Suite) TestSaveWrongHash(t *testing.T) {
func (s *Suite[C]) TestSaveWrongHash(t *testing.T) {
seedRand(t)
b := s.open(t)
@ -679,7 +679,7 @@ func testLoad(b restic.Backend, h restic.Handle) error {
})
}
func (s *Suite) delayedRemove(t testing.TB, be restic.Backend, handles ...restic.Handle) error {
func (s *Suite[C]) delayedRemove(t testing.TB, be restic.Backend, handles ...restic.Handle) error {
// Some backend (swift, I'm looking at you) may implement delayed
// removal of data. Let's wait a bit if this happens.
@ -746,7 +746,7 @@ func delayedList(t testing.TB, b restic.Backend, tpe restic.FileType, max int, m
}
// TestBackend tests all functions of the backend.
func (s *Suite) TestBackend(t *testing.T) {
func (s *Suite[C]) TestBackend(t *testing.T) {
b := s.open(t)
defer s.close(t, b)
@ -867,7 +867,7 @@ func (s *Suite) TestBackend(t *testing.T) {
}
// TestZZZDelete tests the Delete function. The name ensures that this test is executed last.
func (s *Suite) TestZZZDelete(t *testing.T) {
func (s *Suite[C]) TestZZZDelete(t *testing.T) {
if !test.TestCleanupTempDirs {
t.Skipf("not removing backend, TestCleanupTempDirs is false")
}

4
internal/restorer/preallocate_darwin.go → internal/fs/preallocate_darwin.go
View file

@ -1,4 +1,4 @@
package restorer
package fs
import (
"os"
@ -6,7 +6,7 @@ import (
"golang.org/x/sys/unix"
)
func preallocateFile(wr *os.File, size int64) error {
func PreallocateFile(wr *os.File, size int64) error {
// try contiguous first
fst := unix.Fstore_t{
Flags: unix.F_ALLOCATECONTIG | unix.F_ALLOCATEALL,

4
internal/restorer/preallocate_linux.go → internal/fs/preallocate_linux.go
View file

@ -1,4 +1,4 @@
package restorer
package fs
import (
"os"
@ -6,7 +6,7 @@ import (
"golang.org/x/sys/unix"
)
func preallocateFile(wr *os.File, size int64) error {
func PreallocateFile(wr *os.File, size int64) error {
if size <= 0 {
return nil
}

4
internal/restorer/preallocate_other.go → internal/fs/preallocate_other.go
View file

@ -1,11 +1,11 @@
//go:build !linux && !darwin
// +build !linux,!darwin
package restorer
package fs
import "os"
func preallocateFile(wr *os.File, size int64) error {
func PreallocateFile(wr *os.File, size int64) error {
// Maybe truncate can help?
// Windows: This calls SetEndOfFile which preallocates space on disk
return wr.Truncate(size)

7
internal/restorer/preallocate_test.go → internal/fs/preallocate_test.go
View file

@ -1,4 +1,4 @@
package restorer
package fs
import (
"os"
@ -7,7 +7,6 @@ import (
"syscall"
"testing"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/test"
)
@ -23,7 +22,7 @@ func TestPreallocate(t *testing.T) {
test.OK(t, wr.Close())
}()
err = preallocateFile(wr, i)
err = PreallocateFile(wr, i)
if err == syscall.ENOTSUP {
t.SkipNow()
}
@ -32,7 +31,7 @@ func TestPreallocate(t *testing.T) {
fi, err := wr.Stat()
test.OK(t, err)
efi := fs.ExtendedStat(fi)
efi := ExtendedStat(fi)
test.Assert(t, efi.Size == i || efi.Blocks > 0, "Preallocated size of %v, got size %v block %v", i, efi.Size, efi.Blocks)
})
}

5
internal/restic/backend.go
View file

@ -80,3 +80,8 @@ type FileInfo struct {
Size int64
Name string
}
// ApplyEnvironmenter fills in a backend configuration from the environment
type ApplyEnvironmenter interface {
ApplyEnvironment(prefix string) error
}

3
internal/restorer/fileswriter.go
View file

@ -6,6 +6,7 @@ import (
"github.com/cespare/xxhash/v2"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/fs"
)
// writes blobs to target files.
@ -72,7 +73,7 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create
return nil, err
}
} else {
err := preallocateFile(wr.File, createSize)
err := fs.PreallocateFile(wr.File, createSize)
if err != nil {
// Just log the preallocate error but don't let it cause the restore process to fail.
// Preallocate might return an error if the filesystem (implementation) does not

12
internal/ui/backup/progress.go
View file

@ -44,6 +44,7 @@ type Progress struct {
mu sync.Mutex
start time.Time
estimator rateEstimator
scanStarted, scanFinished bool
@ -60,6 +61,7 @@ func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress {
start: time.Now(),
currentFiles: make(map[string]struct{}),
printer: printer,
estimator: *newRateEstimator(time.Now()),
}
p.Updater = *progress.NewUpdater(interval, func(runtime time.Duration, final bool) {
if final {
@ -73,9 +75,14 @@ func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress {
var secondsRemaining uint64
if p.scanFinished {
secs := float64(runtime / time.Second)
rate := p.estimator.rate(time.Now())
tooSlowCutoff := 1024.
if rate <= tooSlowCutoff {
secondsRemaining = 0
} else {
todo := float64(p.total.Bytes - p.processed.Bytes)
secondsRemaining = uint64(secs / float64(p.processed.Bytes) * todo)
secondsRemaining = uint64(todo / rate)
}
}
p.printer.Update(p.total, p.processed, p.errors, p.currentFiles, p.start, secondsRemaining)
@ -105,6 +112,7 @@ func (p *Progress) addProcessed(c Counter) {
p.processed.Files += c.Files
p.processed.Dirs += c.Dirs
p.processed.Bytes += c.Bytes
p.estimator.recordBytes(time.Now(), c.Bytes)
p.scanStarted = true
}

98
internal/ui/backup/rate_estimator.go Normal file
View file

@ -0,0 +1,98 @@
package backup
import (
"container/list"
"time"
)
// rateBucket represents a one second window of recorded progress.
type rateBucket struct {
totalBytes uint64
end time.Time // the end of the time window, exclusive
}
// rateEstimator represents an estimate of the time to complete an operation.
type rateEstimator struct {
buckets *list.List
start time.Time
totalBytes uint64
}
// newRateEstimator returns an estimator initialized to a presumed start time.
func newRateEstimator(start time.Time) *rateEstimator {
return &rateEstimator{buckets: list.New(), start: start}
}
// See trim(), below.
const (
bucketWidth = time.Second
minRateEstimatorBytes = 100 * 1000 * 1000
minRateEstimatorBuckets = 20
minRateEstimatorMinutes = 2
)
// trim removes the oldest history from the estimator assuming a given
// current time.
func (r *rateEstimator) trim(now time.Time) {
// The estimator retains byte transfer counts over a two minute window.
// However, to avoid removing too much history when transfer rates are
// low, the estimator also retains a minimum number of processed bytes
// across a minimum number of buckets. An operation that is processing a
// significant number of bytes per second will typically retain only a
// two minute window's worth of information. One that is making slow
// progress, such as one being over a rate limited connection, typically
// observes bursts of updates as infrequently as every ten or twenty
// seconds, in which case the other limiters will kick in. This heuristic
// avoids wildly fluctuating estimates over rate limited connections.
start := now.Add(-minRateEstimatorMinutes * time.Minute)
for e := r.buckets.Front(); e != nil; e = r.buckets.Front() {
if r.buckets.Len() <= minRateEstimatorBuckets {
break
}
b := e.Value.(*rateBucket)
if b.end.After(start) {
break
}
total := r.totalBytes - b.totalBytes
if total < minRateEstimatorBytes {
break
}
r.start = b.end
r.totalBytes = total
r.buckets.Remove(e)
}
}
// recordBytes records the transfer of a number of bytes at a given
// time. Times passed in successive calls should advance monotonically (as
// is the case with time.Now().
func (r *rateEstimator) recordBytes(now time.Time, bytes uint64) {
if bytes == 0 {
return
}
var tail *rateBucket
if r.buckets.Len() > 0 {
tail = r.buckets.Back().Value.(*rateBucket)
}
if tail == nil || !tail.end.After(now) {
// The new bucket holds measurements in the time range [now .. now+1sec).
tail = &rateBucket{end: now.Add(bucketWidth)}
r.buckets.PushBack(tail)
}
tail.totalBytes += bytes
r.totalBytes += bytes
r.trim(now)
}
// rate returns an estimated bytes per second rate at a given time, or zero
// if there is not enough data to compute a rate.
func (r *rateEstimator) rate(now time.Time) float64 {
r.trim(now)
if !r.start.Before(now) {
return 0
}
elapsed := float64(now.Sub(r.start)) / float64(time.Second)
rate := float64(r.totalBytes) / elapsed
return rate
}

213
internal/ui/backup/rate_estimator_test.go Normal file
View file

@ -0,0 +1,213 @@
package backup
import (
"fmt"
"math"
"testing"
"time"
rtest "github.com/restic/restic/internal/test"
)
const float64EqualityThreshold = 1e-6
func almostEqual(a, b float64) bool {
if math.IsNaN(a) || math.IsNaN(b) {
panic("almostEqual passed a NaN")
}
return math.Abs(a-b) <= float64EqualityThreshold
}
func TestEstimatorDefault(t *testing.T) {
var start time.Time
e := newRateEstimator(start)
r := e.rate(start)
rtest.Assert(t, r == 0, "e.Rate == %v, want zero", r)
r = e.rate(start.Add(time.Hour))
rtest.Assert(t, r == 0, "e.Rate == %v, want zero", r)
}
func TestEstimatorSimple(t *testing.T) {
var start time.Time
type testcase struct {
bytes uint64
when time.Duration
rate float64
}
cases := []testcase{
{0, 0, 0},
{1, time.Second, 1},
{60, time.Second, 60},
{60, time.Minute, 1},
}
for _, c := range cases {
name := fmt.Sprintf("%+v", c)
t.Run(name, func(t *testing.T) {
e := newRateEstimator(start)
e.recordBytes(start.Add(time.Second), c.bytes)
rate := e.rate(start.Add(c.when))
rtest.Assert(t, almostEqual(rate, c.rate), "e.Rate == %v, want %v", rate, c.rate)
})
}
}
func TestBucketWidth(t *testing.T) {
var when time.Time
// Recording byte transfers within a bucket width's time window uses one
// bucket.
e := newRateEstimator(when)
e.recordBytes(when, 1)
e.recordBytes(when.Add(bucketWidth-time.Nanosecond), 1)
rtest.Assert(t, e.buckets.Len() == 1, "e.buckets.Len() is %d, want 1", e.buckets.Len())
b := e.buckets.Back().Value.(*rateBucket)
rtest.Assert(t, b.totalBytes == 2, "b.totalBytes is %d, want 2", b.totalBytes)
rtest.Assert(t, b.end == when.Add(bucketWidth), "b.end is %v, want %v", b.end, when.Add(bucketWidth))
// Recording a byte outside the bucket width causes another bucket.
e.recordBytes(when.Add(bucketWidth), 1)
rtest.Assert(t, e.buckets.Len() == 2, "e.buckets.Len() is %d, want 2", e.buckets.Len())
b = e.buckets.Back().Value.(*rateBucket)
rtest.Assert(t, b.totalBytes == 1, "b.totalBytes is %d, want 1", b.totalBytes)
rtest.Assert(t, b.end == when.Add(2*bucketWidth), "b.end is %v, want %v", b.end, when.Add(bucketWidth))
// Recording a byte after a longer delay creates a sparse bucket list.
e.recordBytes(when.Add(time.Hour+time.Millisecond), 7)
rtest.Assert(t, e.buckets.Len() == 3, "e.buckets.Len() is %d, want 3", e.buckets.Len())
b = e.buckets.Back().Value.(*rateBucket)
rtest.Assert(t, b.totalBytes == 7, "b.totalBytes is %d, want 7", b.totalBytes)
rtest.Equals(t, when.Add(time.Hour+time.Millisecond+time.Second), b.end)
}
type chunk struct {
repetitions uint64 // repetition count
bytes uint64 // byte count (every second)
}
func applyChunks(chunks []chunk, t time.Time, e *rateEstimator) time.Time {
for _, c := range chunks {
for i := uint64(0); i < c.repetitions; i++ {
e.recordBytes(t, c.bytes)
t = t.Add(time.Second)
}
}
return t
}
func TestEstimatorResponsiveness(t *testing.T) {
type testcase struct {
description string
chunks []chunk
rate float64
}
cases := []testcase{
{
"1000 bytes/sec over one second",
[]chunk{
{1, 1000},
},
1000,
},
{
"1000 bytes/sec over one minute",
[]chunk{
{60, 1000},
},
1000,
},
{
"1000 bytes/sec for 10 seconds, then 2000 bytes/sec for 10 seconds",
[]chunk{
{10, 1000},
{10, 2000},
},
1500,
},
{
"1000 bytes/sec for one minute, then 2000 bytes/sec for one minute",
[]chunk{
{60, 1000},
{60, 2000},
},
1500,
},
{
"rate doubles after 30 seconds",
[]chunk{
{30, minRateEstimatorBytes},
{90, 2 * minRateEstimatorBytes},
},
minRateEstimatorBytes * 1.75,
},
{
"rate doubles after 31 seconds",
[]chunk{
{31, minRateEstimatorBytes},
{90, 2 * minRateEstimatorBytes},
},
// The expected rate is the same as the prior test case because the
// first second has rolled off the estimator.
minRateEstimatorBytes * 1.75,
},
{
"rate doubles after 90 seconds",
[]chunk{
{90, minRateEstimatorBytes},
{90, 2 * minRateEstimatorBytes},
},
// The expected rate is the same as the prior test case because the
// first 60 seconds have rolled off the estimator.
minRateEstimatorBytes * 1.75,
},
{
"rate doubles for two full minutes",
[]chunk{
{60, minRateEstimatorBytes},
{120, 2 * minRateEstimatorBytes},
},
2 * minRateEstimatorBytes,
},
{
"rate falls to zero",
[]chunk{
{30, minRateEstimatorBytes},
{30, 0},
},
minRateEstimatorBytes / 2,
},
{
"rate falls to zero for extended time",
[]chunk{
{60, 1000},
{300, 0},
},
1000 * 60 / (60 + 300.0),
},
{
"rate falls to zero for extended time (from high rate)",
[]chunk{
{2 * minRateEstimatorBuckets, minRateEstimatorBytes},
{300, 0},
},
// Expect that only minRateEstimatorBuckets buckets are used in the
// rate estimate.
minRateEstimatorBytes * minRateEstimatorBuckets /
(minRateEstimatorBuckets + 300.0),
},
}
for _, c := range cases {
t.Run(c.description, func(t *testing.T) {
var w time.Time
e := newRateEstimator(w)
w = applyChunks(c.chunks, w, e)
r := e.rate(w)
rtest.Assert(t, almostEqual(r, c.rate), "e.Rate == %f, want %f", r, c.rate)
})
}
}