From 5d28a65fada4c2a217777fac98f851f87a3b7f66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20=C5=9Awi=C4=99cki?= Date: Wed, 29 Mar 2017 23:58:25 +0200 Subject: [PATCH] Implement OpenStack swift backend This commit implements support for OpenStack swift storage server, tested on OVH public cloud storage. Special thanks to jayme-github who helped with the implementation. --- doc/Manual.md | 53 +++ run_integration_tests.go | 81 +++++ src/cmds/restic/global.go | 50 +++ src/restic/backend/location/location.go | 2 + src/restic/backend/location/location_test.go | 19 ++ src/restic/backend/swift/backend_test.go | 87 +++++ src/restic/backend/swift/config.go | 52 +++ src/restic/backend/swift/config_test.go | 50 +++ src/restic/backend/swift/swift.go | 335 +++++++++++++++++++ src/restic/backend/swift/swift_test.go | 76 +++++ src/restic/backend/test/tests.go | 19 +- src/restic/test/vars.go | 2 + 12 files changed, 824 insertions(+), 2 deletions(-) create mode 100644 src/restic/backend/swift/backend_test.go create mode 100644 src/restic/backend/swift/config.go create mode 100644 src/restic/backend/swift/config_test.go create mode 100644 src/restic/backend/swift/swift.go create mode 100644 src/restic/backend/swift/swift_test.go diff --git a/doc/Manual.md b/doc/Manual.md index bbf2e7714..7140d0c67 100644 --- a/doc/Manual.md +++ b/doc/Manual.md @@ -606,6 +606,59 @@ For an S3-compatible server that is not Amazon (like Minio, see below), or is only available via HTTP, you can specify the URL to the server like this: `s3:http://server:port/bucket_name`. +# Create an OpenStack Swift repository + +Restic can backup data to an OpenStack Swift container. Because Swift supports various authentication methods, credentials are passed through environment variables. +In order to help integration with existing openstack installations, the naming convention of those variables follows official python swift client: + +```console +# For keystone v1 authentication +$ export ST_AUTH= +$ export ST_USER= +$ export ST_KEY= + +# For keystone v2 authentication (some variables are optional) +$ export OS_AUTH_URL= +$ export OS_REGION_NAME= +$ export OS_USERNAME= +$ export OS_PASSWORD= +$ export OS_TENANT_ID= +$ export OS_TENANT_NAME= + +# For keystone v3 authentication (some variables are optional) +$ export OS_AUTH_URL= +$ export OS_REGION_NAME= +$ export OS_USERNAME= +$ export OS_PASSWORD= +$ export OS_USER_DOMAIN_NAME= +$ export OS_PROJECT_NAME= +$ export OS_PROJECT_DOMAIN_NAME= + +# For authentication based on tokens +$ export OS_STORAGE_URL= +$ export OS_AUTH_TOKEN= +``` + +Restic should be compatible with [OpenStack RC file](https://docs.openstack.org/user-guide/common/cli-set-environment-variables-using-openstack-rc.html) in most cases. + +Once environment variables are set up, it's easy to create new restic repository. The name of swift container and optional subpath can be specified in the URL. +If the container does not exist, it will be created automatically: + +```console +$ restic -r swift:container_name:/path init # path is optional +enter password for new backend: +enter password again: +created restic backend eefee03bbd at swift:container_name:/path +Please note that knowledge of your password is required to access the repository. +Losing your password means that your data is irrecoverably lost. +``` + +The policy of new container created by restic can be changed using environment variable: + +```console +$ export SWIFT_DEFAULT_CONTAINER_POLICY= +``` + ## Create a Minio Server repository [Minio](https://www.minio.io) is an Open Source Object Storage, written in Go and compatible with AWS S3 API. diff --git a/run_integration_tests.go b/run_integration_tests.go index 53207550c..d88a34125 100644 --- a/run_integration_tests.go +++ b/run_integration_tests.go @@ -5,6 +5,7 @@ package main import ( "bufio" "bytes" + "encoding/json" "errors" "flag" "fmt" @@ -171,6 +172,83 @@ func (env *TravisEnvironment) runRESTServer() error { return nil } +func (env *TravisEnvironment) generateSwiftAuthToken() error { + + authServer := os.Getenv("RESTIC_TEST_SWIFT_AUTH_SERVER") + authUser := os.Getenv("RESTIC_TEST_SWIFT_AUTH_USERNAME") + authPass := os.Getenv("RESTIC_TEST_SWIFT_AUTH_PASSWORD") + authTenant := os.Getenv("RESTIC_TEST_SWIFT_TENANT_NAME") + authRegion := os.Getenv("RESTIC_TEST_SWIFT_REGION_NAME") + + if authServer == "" { + msg("Skipping Swift integration tests - missing Swift server configuration\n") + return nil + } + + msg("Generating Swift auth token\n") + + authRequestBody, err := json.Marshal(map[string]interface{}{ + "auth": map[string]interface{}{ + "tenantName": authTenant, + "passwordCredentials": map[string]interface{}{ + "username": authUser, + "password": authPass, + }, + }, + }) + if err != nil { + return fmt.Errorf("Error building swift auth request body: %v", err) + } + + client := &http.Client{} + req, err := http.NewRequest("POST", authServer+"/tokens", bytes.NewReader(authRequestBody)) + if err != nil { + return fmt.Errorf("Error building request: %v", err) + } + + req.Header.Add("Content-Type", "application/json") + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("Error connecting swift auth server: %v", err) + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("Authentication request failed: %s", resp.Status) + } + + responseBody := map[string]interface{}{} + if err := json.NewDecoder(resp.Body).Decode(&responseBody); err != nil { + return err + } + access := responseBody["access"].(map[string]interface{}) + + // Extract Token + env.env["RESTIC_TEST_SWIFT_TOKEN"] = access["token"].(map[string]interface{})["id"].(string) + + // Extract Storage server URL + for _, service := range access["serviceCatalog"].([]interface{}) { + service := service.(map[string]interface{}) + + // Filter out services other than object-store + if service["type"].(string) != "object-store" { + continue + } + + // Find region we're interested in + for _, endpoint := range service["endpoints"].([]interface{}) { + endpoint := endpoint.(map[string]interface{}) + if endpoint["region"].(string) == authRegion { + env.env["RESTIC_TEST_SWIFT_SERVER"] = endpoint["publicURL"].(string) + } + } + } + + if _, found := env.env["RESTIC_TEST_SWIFT_SERVER"]; !found { + return fmt.Errorf("Could not find suitable storage swift server in region: %s", authRegion) + } + + return nil +} + // Prepare installs dependencies and starts services in order to run the tests. func (env *TravisEnvironment) Prepare() error { env.env = make(map[string]string) @@ -199,6 +277,9 @@ func (env *TravisEnvironment) Prepare() error { if err := env.runRESTServer(); err != nil { return err } + if err := env.generateSwiftAuthToken(); err != nil { + return err + } if *runCrossCompile && !(runtime.Version() < "go1.7") { // only test cross compilation on linux with Travis diff --git a/src/cmds/restic/global.go b/src/cmds/restic/global.go index 45b41cf33..66598955f 100644 --- a/src/cmds/restic/global.go +++ b/src/cmds/restic/global.go @@ -16,6 +16,7 @@ import ( "restic/backend/rest" "restic/backend/s3" "restic/backend/sftp" + "restic/backend/swift" "restic/debug" "restic/options" "restic/repository" @@ -356,6 +357,51 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro debug.Log("opening s3 repository at %#v", cfg) return cfg, nil + case "swift": + cfg := loc.Config.(swift.Config) + + for _, val := range []struct { + s *string + env string + }{ + // v2/v3 specific + {&cfg.UserName, "OS_USERNAME"}, + {&cfg.APIKey, "OS_PASSWORD"}, + {&cfg.Region, "OS_REGION_NAME"}, + {&cfg.AuthURL, "OS_AUTH_URL"}, + + // v3 specific + {&cfg.Domain, "OS_USER_DOMAIN_NAME"}, + {&cfg.Tenant, "OS_PROJECT_NAME"}, + {&cfg.TenantDomain, "OS_PROJECT_DOMAIN_NAME"}, + + // v2 specific + {&cfg.TenantID, "OS_TENANT_ID"}, + {&cfg.Tenant, "OS_TENANT_NAME"}, + + // v1 specific + {&cfg.AuthURL, "ST_AUTH"}, + {&cfg.UserName, "ST_USER"}, + {&cfg.APIKey, "ST_KEY"}, + + // Manual authentication + {&cfg.StorageURL, "OS_STORAGE_URL"}, + {&cfg.AuthToken, "OS_AUTH_TOKEN"}, + + {&cfg.DefaultContainerPolicy, "SWIFT_DEFAULT_CONTAINER_POLICY"}, + } { + if *val.s == "" { + *val.s = os.Getenv(val.env) + } + } + + if err := opts.Apply(loc.Scheme, &cfg); err != nil { + return nil, err + } + + debug.Log("opening swift repository at %#v", cfg) + return cfg, nil + case "rest": cfg := loc.Config.(rest.Config) if err := opts.Apply(loc.Scheme, &cfg); err != nil { @@ -391,6 +437,8 @@ func open(s string, opts options.Options) (restic.Backend, error) { be, err = sftp.OpenWithConfig(cfg.(sftp.Config)) case "s3": be, err = s3.Open(cfg.(s3.Config)) + case "swift": + be, err = swift.Open(cfg.(swift.Config)) case "rest": be, err = rest.Open(cfg.(rest.Config)) @@ -425,6 +473,8 @@ func create(s string, opts options.Options) (restic.Backend, error) { return sftp.CreateWithConfig(cfg.(sftp.Config)) case "s3": return s3.Open(cfg.(s3.Config)) + case "swift": + return swift.Open(cfg.(swift.Config)) case "rest": return rest.Create(cfg.(rest.Config)) } diff --git a/src/restic/backend/location/location.go b/src/restic/backend/location/location.go index 23e0af37b..d730c17d4 100644 --- a/src/restic/backend/location/location.go +++ b/src/restic/backend/location/location.go @@ -8,6 +8,7 @@ import ( "restic/backend/rest" "restic/backend/s3" "restic/backend/sftp" + "restic/backend/swift" ) // Location specifies the location of a repository, including the method of @@ -28,6 +29,7 @@ var parsers = []parser{ {"local", local.ParseConfig}, {"sftp", sftp.ParseConfig}, {"s3", s3.ParseConfig}, + {"swift", swift.ParseConfig}, {"rest", rest.ParseConfig}, } diff --git a/src/restic/backend/location/location_test.go b/src/restic/backend/location/location_test.go index fe07ee506..a79a60e9a 100644 --- a/src/restic/backend/location/location_test.go +++ b/src/restic/backend/location/location_test.go @@ -9,6 +9,7 @@ import ( "restic/backend/rest" "restic/backend/s3" "restic/backend/sftp" + "restic/backend/swift" ) func parseURL(s string) *url.URL { @@ -195,6 +196,24 @@ var parseTests = []struct { }, }, }, + { + "swift:container17:/", + Location{Scheme: "swift", + Config: swift.Config{ + Container: "container17", + Prefix: "", + }, + }, + }, + { + "swift:container17:/prefix97", + Location{Scheme: "swift", + Config: swift.Config{ + Container: "container17", + Prefix: "prefix97", + }, + }, + }, { "rest:http://hostname.foo:1234/", Location{Scheme: "rest", diff --git a/src/restic/backend/swift/backend_test.go b/src/restic/backend/swift/backend_test.go new file mode 100644 index 000000000..b4409d145 --- /dev/null +++ b/src/restic/backend/swift/backend_test.go @@ -0,0 +1,87 @@ +// DO NOT EDIT, AUTOMATICALLY GENERATED +package swift_test + +import ( + "testing" + + "restic/backend/test" +) + +var SkipMessage string + +func TestSwiftBackendCreate(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestCreate(t) +} + +func TestSwiftBackendOpen(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestOpen(t) +} + +func TestSwiftBackendCreateWithConfig(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestCreateWithConfig(t) +} + +func TestSwiftBackendLocation(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestLocation(t) +} + +func TestSwiftBackendConfig(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestConfig(t) +} + +func TestSwiftBackendLoad(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestLoad(t) +} + +func TestSwiftBackendSave(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSave(t) +} + +func TestSwiftBackendSaveFilenames(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSaveFilenames(t) +} + +func TestSwiftBackendBackend(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestBackend(t) +} + +func TestSwiftBackendDelete(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestDelete(t) +} + +func TestSwiftBackendCleanup(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestCleanup(t) +} diff --git a/src/restic/backend/swift/config.go b/src/restic/backend/swift/config.go new file mode 100644 index 000000000..8178f00e3 --- /dev/null +++ b/src/restic/backend/swift/config.go @@ -0,0 +1,52 @@ +package swift + +import ( + "net/url" + "regexp" + "restic/errors" +) + +var ( + urlParser = regexp.MustCompile("^([^:]+):/(.*)$") +) + +// Config contains basic configuration needed to specify swift location for a swift server +type Config struct { + UserName string + Domain string + APIKey string + AuthURL string + Region string + Tenant string + TenantID string + TenantDomain string + TrustID string + + StorageURL string + AuthToken string + + Container string + Prefix string + DefaultContainerPolicy string +} + +// ParseConfig parses the string s and extract swift's container name and prefix. +func ParseConfig(s string) (interface{}, error) { + + url, err := url.Parse(s) + if err != nil { + return nil, errors.Wrap(err, "url.Parse") + } + + m := urlParser.FindStringSubmatch(url.Opaque) + if len(m) == 0 { + return nil, errors.New("swift: invalid URL, valid syntax is: 'swift:container-name:/[optional-prefix]'") + } + + cfg := Config{ + Container: m[1], + Prefix: m[2], + } + + return cfg, nil +} diff --git a/src/restic/backend/swift/config_test.go b/src/restic/backend/swift/config_test.go new file mode 100644 index 000000000..97ec8d9c6 --- /dev/null +++ b/src/restic/backend/swift/config_test.go @@ -0,0 +1,50 @@ +package swift + +import "testing" + +var configTests = []struct { + s string + cfg Config +}{ + {"swift:cnt1:/", Config{Container: "cnt1", Prefix: ""}}, + {"swift:cnt2:/prefix", Config{Container: "cnt2", Prefix: "prefix"}}, + {"swift:cnt3:/prefix/longer", Config{Container: "cnt3", Prefix: "prefix/longer"}}, + {"swift:cnt4:/prefix?params", Config{Container: "cnt4", Prefix: "prefix"}}, + {"swift:cnt5:/prefix#params", Config{Container: "cnt5", Prefix: "prefix"}}, +} + +func TestParseConfigInternal(t *testing.T) { + for i, test := range configTests { + cfg, err := ParseConfig(test.s) + if err != nil { + t.Errorf("test %d:%s failed: %v", i, test.s, err) + continue + } + + if cfg != test.cfg { + t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v", + i, test.s, test.cfg, cfg) + continue + } + } +} + +var configTestsInvalid = []string{ + "swift://hostname/container", + "swift:////", + "swift://", + "swift:////prefix", + "swift:container", + "swift:container:", + "swift:container/prefix", +} + +func TestParseConfigInvalid(t *testing.T) { + for i, test := range configTestsInvalid { + _, err := ParseConfig(test) + if err == nil { + t.Errorf("test %d: invalid config %s did not return an error", i, test) + continue + } + } +} diff --git a/src/restic/backend/swift/swift.go b/src/restic/backend/swift/swift.go new file mode 100644 index 000000000..3d6bffe0f --- /dev/null +++ b/src/restic/backend/swift/swift.go @@ -0,0 +1,335 @@ +package swift + +import ( + "io" + "path" + "restic" + "restic/backend" + "restic/debug" + "restic/errors" + "strings" + "time" + + "github.com/ncw/swift" +) + +const connLimit = 10 + +// beSwift is a backend which stores the data on a swift endpoint. +type beSwift struct { + conn *swift.Connection + connChan chan struct{} + container string // Container name + prefix string // Prefix of object names in the container +} + +// Open opens the swift backend at a container in region. The container is +// created if it does not exist yet. +func Open(cfg Config) (restic.Backend, error) { + + be := &beSwift{ + conn: &swift.Connection{ + UserName: cfg.UserName, + Domain: cfg.Domain, + ApiKey: cfg.APIKey, + AuthUrl: cfg.AuthURL, + Region: cfg.Region, + Tenant: cfg.Tenant, + TenantId: cfg.TenantID, + TenantDomain: cfg.TenantDomain, + TrustId: cfg.TrustID, + StorageUrl: cfg.StorageURL, + AuthToken: cfg.AuthToken, + ConnectTimeout: time.Minute, + Timeout: time.Minute, + }, + container: cfg.Container, + prefix: cfg.Prefix, + } + be.createConnections() + + // Authenticate if needed + if !be.conn.Authenticated() { + if err := be.conn.Authenticate(); err != nil { + return nil, errors.Wrap(err, "conn.Authenticate") + } + } + + // Ensure container exists + switch _, _, err := be.conn.Container(be.container); err { + case nil: + // Container exists + + case swift.ContainerNotFound: + err = be.createContainer(cfg.DefaultContainerPolicy) + if err != nil { + return nil, errors.Wrap(err, "beSwift.createContainer") + } + + default: + return nil, errors.Wrap(err, "conn.Container") + } + + return be, nil +} + +func (be *beSwift) swiftpath(h restic.Handle) string { + + var dir string + + switch h.Type { + case restic.ConfigFile: + dir = "" + h.Name = backend.Paths.Config + case restic.DataFile: + dir = backend.Paths.Data + case restic.SnapshotFile: + dir = backend.Paths.Snapshots + case restic.IndexFile: + dir = backend.Paths.Index + case restic.LockFile: + dir = backend.Paths.Locks + case restic.KeyFile: + dir = backend.Paths.Keys + default: + dir = string(h.Type) + } + + return path.Join(be.prefix, dir, h.Name) +} + +func (be *beSwift) createConnections() { + be.connChan = make(chan struct{}, connLimit) + for i := 0; i < connLimit; i++ { + be.connChan <- struct{}{} + } +} + +func (be *beSwift) createContainer(policy string) error { + var h swift.Headers + if policy != "" { + h = swift.Headers{ + "X-Storage-Policy": policy, + } + } + + return be.conn.ContainerCreate(be.container, h) +} + +// Location returns this backend's location (the container name). +func (be *beSwift) Location() string { + return be.container +} + +// Load returns a reader that yields the contents of the file at h at the +// given offset. If length is nonzero, only a portion of the file is +// returned. rd must be closed after use. +func (be *beSwift) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + debug.Log("Load %v, length %v, offset %v", h, length, offset) + if err := h.Valid(); err != nil { + return nil, err + } + + if offset < 0 { + return nil, errors.New("offset is negative") + } + + if length < 0 { + return nil, errors.Errorf("invalid length %d", length) + } + + objName := be.swiftpath(h) + + <-be.connChan + defer func() { + be.connChan <- struct{}{} + }() + + obj, _, err := be.conn.ObjectOpen(be.container, objName, false, nil) + if err != nil { + debug.Log(" err %v", err) + return nil, errors.Wrap(err, "conn.ObjectOpen") + } + + // if we're going to read the whole object, just pass it on. + if length == 0 { + debug.Log("Load %v: pass on object", h) + _, err = obj.Seek(offset, 0) + if err != nil { + _ = obj.Close() + return nil, errors.Wrap(err, "obj.Seek") + } + + return obj, nil + } + + // otherwise pass a LimitReader + size, err := obj.Length() + if err != nil { + return nil, errors.Wrap(err, "obj.Length") + } + + if offset > size { + _ = obj.Close() + return nil, errors.Errorf("offset larger than file size") + } + + _, err = obj.Seek(offset, 0) + if err != nil { + _ = obj.Close() + return nil, errors.Wrap(err, "obj.Seek") + } + + return backend.LimitReadCloser(obj, int64(length)), nil +} + +// Save stores data in the backend at the handle. +func (be *beSwift) Save(h restic.Handle, rd io.Reader) (err error) { + if err = h.Valid(); err != nil { + return err + } + + debug.Log("Save %v", h) + + objName := be.swiftpath(h) + + // Check key does not already exist + switch _, _, err = be.conn.Object(be.container, objName); err { + case nil: + debug.Log("%v already exists", h) + return errors.New("key already exists") + + case swift.ObjectNotFound: + // Ok, that's what we want + + default: + return errors.Wrap(err, "conn.Object") + } + + <-be.connChan + defer func() { + be.connChan <- struct{}{} + }() + + encoding := "binary/octet-stream" + + debug.Log("PutObject(%v, %v, %v)", + be.container, objName, encoding) + //err = be.conn.ObjectPutBytes(be.container, objName, p, encoding) + _, err = be.conn.ObjectPut(be.container, objName, rd, true, "", encoding, nil) + debug.Log("%v, err %#v", objName, err) + + return errors.Wrap(err, "client.PutObject") +} + +// Stat returns information about a blob. +func (be *beSwift) Stat(h restic.Handle) (bi restic.FileInfo, err error) { + debug.Log("%v", h) + + objName := be.swiftpath(h) + + obj, _, err := be.conn.Object(be.container, objName) + if err != nil { + debug.Log("Object() err %v", err) + return restic.FileInfo{}, errors.Wrap(err, "conn.Object") + } + + return restic.FileInfo{Size: obj.Bytes}, nil +} + +// Test returns true if a blob of the given type and name exists in the backend. +func (be *beSwift) Test(h restic.Handle) (bool, error) { + objName := be.swiftpath(h) + switch _, _, err := be.conn.Object(be.container, objName); err { + case nil: + return true, nil + + case swift.ObjectNotFound: + return false, nil + + default: + return false, errors.Wrap(err, "conn.Object") + } +} + +// Remove removes the blob with the given name and type. +func (be *beSwift) Remove(h restic.Handle) error { + objName := be.swiftpath(h) + err := be.conn.ObjectDelete(be.container, objName) + debug.Log("Remove(%v) -> err %v", h, err) + return errors.Wrap(err, "conn.ObjectDelete") +} + +// List returns a channel that yields all names of blobs of type t. A +// goroutine is started for this. If the channel done is closed, sending +// stops. +func (be *beSwift) List(t restic.FileType, done <-chan struct{}) <-chan string { + debug.Log("listing %v", t) + ch := make(chan string) + + prefix := be.swiftpath(restic.Handle{Type: t}) + "/" + + go func() { + defer close(ch) + + be.conn.ObjectsWalk(be.container, &swift.ObjectsOpts{Prefix: prefix}, + func(opts *swift.ObjectsOpts) (interface{}, error) { + newObjects, err := be.conn.ObjectNames(be.container, opts) + if err != nil { + return nil, errors.Wrap(err, "conn.ObjectNames") + } + for _, obj := range newObjects { + m := strings.TrimPrefix(obj, prefix) + if m == "" { + continue + } + + select { + case ch <- m: + case <-done: + return nil, io.EOF + } + } + return newObjects, nil + }) + }() + + return ch +} + +// Remove keys for a specified backend type. +func (be *beSwift) removeKeys(t restic.FileType) error { + done := make(chan struct{}) + defer close(done) + for key := range be.List(restic.DataFile, done) { + err := be.Remove(restic.Handle{Type: restic.DataFile, Name: key}) + if err != nil { + return err + } + } + + return nil +} + +// Delete removes all restic objects in the container. +// It will not remove the container itself. +func (be *beSwift) Delete() error { + alltypes := []restic.FileType{ + restic.DataFile, + restic.KeyFile, + restic.LockFile, + restic.SnapshotFile, + restic.IndexFile} + + for _, t := range alltypes { + err := be.removeKeys(t) + if err != nil { + return nil + } + } + + return be.Remove(restic.Handle{Type: restic.ConfigFile}) +} + +// Close does nothing +func (be *beSwift) Close() error { return nil } diff --git a/src/restic/backend/swift/swift_test.go b/src/restic/backend/swift/swift_test.go new file mode 100644 index 000000000..2f3976b0c --- /dev/null +++ b/src/restic/backend/swift/swift_test.go @@ -0,0 +1,76 @@ +package swift_test + +import ( + "fmt" + "math/rand" + "restic" + "time" + + "restic/errors" + + "restic/backend/swift" + "restic/backend/test" + . "restic/test" + + swiftclient "github.com/ncw/swift" +) + +//go:generate go run ../test/generate_backend_tests.go + +func init() { + if TestSwiftServer == "" { + SkipMessage = "swift test server not available" + return + } + + // Generate random container name to allow simultaneous test + // on the same swift backend + containerName := fmt.Sprintf( + "restictestcontainer_%d_%d", + time.Now().Unix(), + rand.Uint32(), + ) + + cfg := swift.Config{ + Container: containerName, + StorageURL: TestSwiftServer, + AuthToken: TestSwiftToken, + } + + test.CreateFn = func() (restic.Backend, error) { + be, err := swift.Open(cfg) + if err != nil { + return nil, err + } + + exists, err := be.Test(restic.Handle{Type: restic.ConfigFile}) + if err != nil { + return nil, err + } + + if exists { + return nil, errors.New("config already exists") + } + + return be, nil + } + + test.OpenFn = func() (restic.Backend, error) { + return swift.Open(cfg) + } + + test.CleanupFn = func() error { + client := swiftclient.Connection{ + StorageUrl: TestSwiftServer, + AuthToken: TestSwiftToken, + } + objects, err := client.ObjectsAll(containerName, nil) + if err != nil { + return err + } + for _, o := range objects { + client.ObjectDelete(containerName, o.Name) + } + return client.ContainerDelete(containerName) + } +} diff --git a/src/restic/backend/test/tests.go b/src/restic/backend/test/tests.go index f2e6b2820..1aff0e4a3 100644 --- a/src/restic/backend/test/tests.go +++ b/src/restic/backend/test/tests.go @@ -13,6 +13,7 @@ import ( "sort" "strings" "testing" + "time" "restic/test" @@ -431,6 +432,20 @@ func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) res return h } +func delayedRemove(b restic.Backend, h restic.Handle) error { + // Some backend (swift, I'm looking at you) may implement delayed + // removal of data. Let's wait a bit if this happens. + err := b.Remove(h) + found, err := b.Test(h) + for i := 0; found && i < 10; i++ { + found, err = b.Test(h) + if found { + time.Sleep(100 * time.Millisecond) + } + } + return err +} + // TestBackend tests all functions of the backend. func TestBackend(t testing.TB) { b := open(t) @@ -505,7 +520,7 @@ func TestBackend(t testing.TB) { test.Assert(t, err != nil, "expected error for %v, got %v", h, err) // remove and recreate - err = b.Remove(h) + err = delayedRemove(b, h) test.OK(t, err) // test that the blob is gone @@ -554,7 +569,7 @@ func TestBackend(t testing.TB) { found, err := b.Test(h) test.OK(t, err) - test.OK(t, b.Remove(h)) + test.OK(t, delayedRemove(b, h)) found, err = b.Test(h) test.OK(t, err) diff --git a/src/restic/test/vars.go b/src/restic/test/vars.go index bb9f6b13d..4699821b4 100644 --- a/src/restic/test/vars.go +++ b/src/restic/test/vars.go @@ -15,6 +15,8 @@ var ( TestWalkerPath = getStringVar("RESTIC_TEST_PATH", ".") BenchArchiveDirectory = getStringVar("RESTIC_BENCH_DIR", ".") TestS3Server = getStringVar("RESTIC_TEST_S3_SERVER", "") + TestSwiftServer = getStringVar("RESTIC_TEST_SWIFT_SERVER", "") + TestSwiftToken = getStringVar("RESTIC_TEST_SWIFT_TOKEN", "") TestRESTServer = getStringVar("RESTIC_TEST_REST_SERVER", "") )