From faca6854174b1d74c30c48cee8540cd621b8d22e Mon Sep 17 00:00:00 2001 From: Chapuis Bertil Date: Thu, 6 Aug 2015 12:08:55 +0200 Subject: [PATCH] simplified the backend and added documentation --- backend/backend_test.go | 1 - backend/rest/rest.go | 71 ++++++++++++++++++------------------ backend/rest_test.go | 79 +++++++++++++---------------------------- backend/s3/s3.go | 10 ++++-- cmd/restic/global.go | 3 ++ doc/REST_backend.md | 55 ++++++++++++++++++++++++++++ 6 files changed, 128 insertions(+), 91 deletions(-) create mode 100644 doc/REST_backend.md diff --git a/backend/backend_test.go b/backend/backend_test.go index 311a7b2d6..0ff5f01b4 100644 --- a/backend/backend_test.go +++ b/backend/backend_test.go @@ -128,7 +128,6 @@ func testBackend(b backend.Backend, t *testing.T) { OK(t, err) found, err := b.Test(tpe, id.String()) - OK(t, err) Assert(t, found, fmt.Sprintf("id %q was not found before removal", id)) diff --git a/backend/rest/rest.go b/backend/rest/rest.go index 47b2429c7..27c3de9a2 100644 --- a/backend/rest/rest.go +++ b/backend/rest/rest.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "errors" - "fmt" "io" "io/ioutil" "net/http" @@ -55,22 +54,31 @@ func (rb *RestBlob) Close() error { // Finalize moves the data blob to the final location for type and name. func (rb *RestBlob) Finalize(t backend.Type, name string) error { if rb.final { - return errors.New("Already finalized") + return errors.New("already finalized") } rb.final = true - // Check key does not already exist - resp, err := http.Get(restPath(rb.b.url, t, name)) - if err == nil && resp.StatusCode == 200 { - return errors.New("Key already exists") + // Check key does not already exist. + req, err := http.NewRequest("HEAD", restPath(rb.b.url, t, name), nil) + if err != nil { + return err + } + + resp, errh := http.DefaultClient.Do(req) + if errh != nil { + return errh + } + if resp.StatusCode == 200 { + return errors.New("key already exists") } <-rb.b.connChan - _, err2 := http.Post(restPath(rb.b.url, t, name), "binary/octet-stream", rb.buf) + _, errp := http.Post(restPath(rb.b.url, t, name), "binary/octet-stream", rb.buf) rb.b.connChan <- struct{}{} rb.buf.Reset() - return err2 + + return errp } // Size returns the number of bytes written to the backend so far. @@ -112,7 +120,7 @@ func (b *Rest) Create() (backend.Blob, error) { func (b *Rest) Get(t backend.Type, name string) (io.ReadCloser, error) { resp, err := http.Get(restPath(b.url, t, name)) if err == nil && resp.StatusCode != 200 { - err = errors.New("Not found") + err = errors.New("blob not found") } return resp.Body, err } @@ -120,39 +128,35 @@ func (b *Rest) Get(t backend.Type, name string) (io.ReadCloser, error) { // GetReader returns an io.ReadCloser for the Blob with the given name of // type t at offset and length. func (b *Rest) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { - rc, err := b.Get(t, name) + req, err := http.NewRequest("GET", restPath(b.url, t, name), nil) if err != nil { return nil, err } - n, errc := io.CopyN(ioutil.Discard, rc, int64(offset)) - if errc != nil { - return nil, errc - } else if n != int64(offset) { - return nil, fmt.Errorf("less bytes read than expected, read: %d, expected: %d", n, offset) + req.Header.Add("Range", "bytes="+string(offset)+"-"+string(offset+length)) + + resp, errg := http.DefaultClient.Do(req) + if errg != nil { + return nil, errg } - if length == 0 { - return rc, nil - } - - return backend.LimitReadCloser(rc, int64(length)), nil + return backend.LimitReadCloser(resp.Body, int64(length)), nil } // Test a boolean value whether a Blob with the name and type exists. func (b *Rest) Test(t backend.Type, name string) (bool, error) { found := false - req, e1 := http.NewRequest("HEAD", restPath(b.url, t, name), nil) - if e1 != nil { - return found, e1 + req, err := http.NewRequest("HEAD", restPath(b.url, t, name), nil) + if err != nil { + return found, err } - resp, err := http.DefaultClient.Do(req) + resp, errh := http.DefaultClient.Do(req) if resp.StatusCode == 200 { found = true } - return found, err + return found, errh } // Remove removes a Blob with type t and name. @@ -162,8 +166,8 @@ func (b *Rest) Remove(t backend.Type, name string) error { return err } - _, err2 := http.DefaultClient.Do(req) - return err2 + _, errd := http.DefaultClient.Do(req) + return errd } // Close the backend @@ -177,21 +181,21 @@ func (b *Rest) Close() error { func (b *Rest) List(t backend.Type, done <-chan struct{}) <-chan string { ch := make(chan string) - resp, e1 := http.Get(restPath(b.url, t, "")) - if e1 != nil { + resp, err := http.Get(restPath(b.url, t, "")) + if err != nil { close(ch) return ch } - data, e2 := ioutil.ReadAll(resp.Body) - if e2 != nil { + data, errd := ioutil.ReadAll(resp.Body) + if errd != nil { close(ch) return ch } var list []string - e3 := json.Unmarshal(data, &list) - if e3 != nil { + errj := json.Unmarshal(data, &list) + if errj != nil { close(ch) return ch } @@ -199,7 +203,6 @@ func (b *Rest) List(t backend.Type, done <-chan struct{}) <-chan string { go func() { defer close(ch) for _, m := range list { - fmt.Println(m) select { case ch <- m: case <-done: diff --git a/backend/rest_test.go b/backend/rest_test.go index ae7b18ce9..0a2152128 100644 --- a/backend/rest_test.go +++ b/backend/rest_test.go @@ -2,7 +2,6 @@ package backend_test import ( "encoding/json" - "fmt" "io/ioutil" "net/http" "net/http/httptest" @@ -10,6 +9,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/gorilla/mux" "github.com/restic/restic/backend" @@ -23,7 +23,8 @@ func setupRestBackend(t *testing.T) *rest.Rest { } func TestRestBackend(t *testing.T) { - // Initializing a temporary repository for the backend + + // Initializing a temporary direcory for the rest backend. path, _ := ioutil.TempDir("", "restic-repository-") defer os.RemoveAll(path) @@ -41,52 +42,33 @@ func TestRestBackend(t *testing.T) { os.MkdirAll(d, backend.Modes.Dir) } - // Routing the repository requests + // Initialize the router for the repository requests. r := mux.NewRouter() - // Exists - r.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + // Check if the repository has already been initialized. + r.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) { file := filepath.Join(path, "config") if _, err := os.Stat(file); err != nil { - http.Error(w, "No repository here", 404) + http.Error(w, "Repository not found", 404) } }).Methods("HEAD") - // List blobs - r.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + // Get the configuration of the repository. + r.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) { file := filepath.Join(path, "config") - if _, err := os.Stat(file); err != nil { - http.Error(w, "No repository here", 404) - } - }).Methods("GET") - - // Head file - r.HandleFunc("/{file}", func(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - file := filepath.Join(path, vars["file"]) - if _, err := os.Stat(file); err != nil { - http.Error(w, "File not found", 404) - } - }).Methods("HEAD") - - // Get file - r.HandleFunc("/{file}", func(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - file := filepath.Join(path, vars["file"]) if _, err := os.Stat(file); err == nil { bytes, _ := ioutil.ReadFile(file) w.Write(bytes) } else { - http.Error(w, "File not found", 404) + http.Error(w, "Repository not found", 404) } }).Methods("GET") - // Put file - r.HandleFunc("/{file}", func(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - file := filepath.Join(path, vars["file"]) + // Initialize the repository and save the configuration. + r.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) { + file := filepath.Join(path, "config") if _, err := os.Stat(file); err == nil { - fmt.Fprintf(w, "Blob already uploaded", 404) + http.Error(w, "Repository already initialized", 403) } else { bytes, _ := ioutil.ReadAll(r.Body) ioutil.WriteFile(file, bytes, 0600) @@ -94,18 +76,7 @@ func TestRestBackend(t *testing.T) { } }).Methods("POST") - // Delete file - r.HandleFunc("/{file}", func(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - file := filepath.Join(path, vars["file"]) - if _, err := os.Stat(file); err == nil { - os.Remove(file) - } else { - fmt.Fprintf(w, "File not found", 404) - } - }).Methods("DELETE") - - // List blobs + // List the blobs of a given type. r.HandleFunc("/{type}/", func(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) path := filepath.Join(path, vars["type"]) @@ -118,50 +89,50 @@ func TestRestBackend(t *testing.T) { w.Write(data) }).Methods("GET") - // Head blob + // Check if a blob of a given type exists. r.HandleFunc("/{type}/{blob}", func(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) blob := filepath.Join(path, vars["type"], vars["blob"]) if _, err := os.Stat(blob); err != nil { - http.Error(w, "File not found", 404) + http.Error(w, "Blob not found", 404) } }).Methods("HEAD") - // Get blob + // Get a blob of a given type. r.HandleFunc("/{type}/{blob}", func(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) blob := filepath.Join(path, vars["type"], vars["blob"]) - if _, err := os.Stat(blob); err == nil { - bytes, _ := ioutil.ReadFile(blob) - w.Write(bytes) + if file, err := os.Open(blob); err == nil { + http.ServeContent(w, r, "", time.Unix(0, 0), file) } else { http.Error(w, "Blob not found", 404) } }).Methods("GET") - // Put blob + // Save a blob of a given type. r.HandleFunc("/{type}/{blob}", func(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) blob := filepath.Join(path, vars["type"], vars["blob"]) if _, err := os.Stat(blob); err == nil { - fmt.Fprintf(w, "Blob already uploaded", 404) + http.Error(w, "Blob already uploaded", 403) } else { bytes, _ := ioutil.ReadAll(r.Body) ioutil.WriteFile(blob, bytes, 0600) } }).Methods("POST") - // Delete blob + // Delete a blob of a given type. r.HandleFunc("/{type}/{blob}", func(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) blob := filepath.Join(path, vars["type"], vars["blob"]) if _, err := os.Stat(blob); err == nil { os.Remove(blob) } else { - fmt.Fprintf(w, "Blob not found", 404) + http.Error(w, "Blob not found", 404) } }).Methods("DELETE") + // Start the server and launch the tests. s := httptest.NewServer(r) defer s.Close() diff --git a/backend/s3/s3.go b/backend/s3/s3.go index edd8cafef..e9e0c0120 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -71,7 +71,7 @@ type s3Blob struct { func (bb *s3Blob) Write(p []byte) (int, error) { if bb.final { - return 0, errors.New("Blob already closed") + return 0, errors.New("blob already closed") } n, err := bb.buf.Write(p) @@ -94,7 +94,7 @@ func (bb *s3Blob) Size() uint { func (bb *s3Blob) Finalize(t backend.Type, name string) error { if bb.final { - return errors.New("Already finalized") + return errors.New("already finalized") } bb.final = true @@ -102,9 +102,15 @@ func (bb *s3Blob) Finalize(t backend.Type, name string) error { path := s3path(t, name) // Check key does not already exist +<<<<<<< HEAD _, err := bb.b.bucket.GetReader(path) if err == nil { return errors.New("key already exists!") +======= + key, err := bb.b.bucket.GetKey(path) + if err == nil && key.Key == path { + return errors.New("key already exists") +>>>>>>> integrated the comments from the review, simplified the backend and added some documentation } <-bb.b.connChan diff --git a/cmd/restic/global.go b/cmd/restic/global.go index a612849e1..f8e1beecf 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -136,12 +136,15 @@ func (o GlobalOptions) OpenRepository() (*repository.Repository, error) { // * sftp://host//tmp/backup -> remote sftp repository on host at path /tmp/backup // * http://host -> remote and public http repository func open(u string) (backend.Backend, error) { +<<<<<<< HEAD // check if the url is a directory that exists fi, err := os.Stat(u) if err == nil && fi.IsDir() { return local.Open(u) } +======= +>>>>>>> integrated the comments from the review, simplified the backend and added some documentation url, err := url.Parse(u) if err != nil { return nil, err diff --git a/doc/REST_backend.md b/doc/REST_backend.md new file mode 100644 index 000000000..91221c785 --- /dev/null +++ b/doc/REST_backend.md @@ -0,0 +1,55 @@ +REST Backend +============ + +Restic can interact with HTTP Backend that respects the following REST API. + +## HEAD /config + +Returns "200 OK" if the repository has already been initialized, +"404 Repository not found" otherwise. + +## GET /config + +Returns the configuration if the repository has already been initialized, +"404 Repository not found" otherwise. + +Response format: binary/octet-stream + +## POST /config + +Saves the configuration transmitted in the request body. +Returns "200 OK" if the configuration has been saved, +"403 Repository already initialized" if the repository has already been initialized. + +Response format: text + +## GET /{type}/ + +Returns a JSON array containing the IDs of all the blobs stored for a given type. + +Response format: JSON + +## HEAD /{type}/{blobID} + +Returns "200 OK" if the repository contains a blob with the given ID and type, +"404 Blob not found" otherwise. + +## GET /{type}/{blobID} + +Returns the content of the blob with the given ID and type, +"404 Blob not found" otherwise. + +Response format: binary/octet-stream + +## POST /{type}/{blobID} + +Saves the content of the request body in a blob with the given ID and type, +"403 Blob already exists" if a blob has already been saved. + +Request format: binary/octet-stream + +## DELETE /{type}/{blobID} + +Deletes the blob with the given ID and type. +Returns "200 OK" if the given blob exists and has been deleted, +"404 Blob not found" otherwise.