diff --git a/src/cmds/restic/global.go b/src/cmds/restic/global.go index 49a2ace30..ff337bd6a 100644 --- a/src/cmds/restic/global.go +++ b/src/cmds/restic/global.go @@ -10,6 +10,7 @@ import ( "restic/backend" "restic/backend/local" + "restic/backend/onedrive" "restic/backend/rest" "restic/backend/s3" "restic/backend/sftp" @@ -251,6 +252,9 @@ func open(s string) (backend.Backend, error) { return s3.Open(cfg) case "rest": return rest.Open(loc.Config.(rest.Config)) + case "onedrive": + debug.Log("open", "opening ondrive repository") + return onedrive.Open(loc.Config.(onedrive.Config)) } debug.Log("open", "invalid repository location: %v", s) @@ -286,6 +290,9 @@ func create(s string) (backend.Backend, error) { return s3.Open(cfg) case "rest": return rest.Open(loc.Config.(rest.Config)) + case "onedrive": + debug.Log("open", "opening ondrive repository") + return onedrive.Open(loc.Config.(onedrive.Config)) } debug.Log("open", "invalid repository scheme: %v", s) diff --git a/src/restic/backend/onedrive/config.go b/src/restic/backend/onedrive/config.go new file mode 100644 index 000000000..1a73ced17 --- /dev/null +++ b/src/restic/backend/onedrive/config.go @@ -0,0 +1,45 @@ +package onedrive + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "strings" + + "golang.org/x/oauth2" +) + +// Config contains all configuration necessary to connect to onedrive. +type Config struct { + Token oauth2.Token `json:"token"` + ClientID string `json:"client_id"` + ClientSecret string `json:"client_secret"` +} + +// ParseConfig parses the string s and extracts the onedrive config. The +// supported configuration format is onedrive:configfile.json +func ParseConfig(s string) (interface{}, error) { + if strings.HasPrefix(s, "onedrive:") { + s = s[9:] + + data := strings.SplitN(s, ":", 1) + if len(data) != 1 { + return nil, errors.New("onedrive: invalid format") + } + + file, err := ioutil.ReadFile(data[0]) + if err != nil { + return nil, fmt.Errorf("onedrive: reading tokenfile (%s) failed with '%s'", data[0], err) + } + + var cfg Config + if err := json.Unmarshal(file, &cfg); err != nil { + return nil, fmt.Errorf("onedrive: token de-serialization failed with '%s'", err) + } + + return cfg, nil + } + + return nil, errors.New("onedrive: invalid format") +} diff --git a/src/restic/backend/onedrive/onedrive.go b/src/restic/backend/onedrive/onedrive.go new file mode 100644 index 000000000..84f4e1676 --- /dev/null +++ b/src/restic/backend/onedrive/onedrive.go @@ -0,0 +1,311 @@ +package onedrive + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + + "golang.org/x/oauth2" + + "restic/backend" + "restic/debug" +) + +const connLimit = 10 +const onedriveBase = "https://api.onedrive.com/v1.0/drive/special/approot:/" + +type onedrive struct { + client *http.Client + connChan chan struct{} +} + +var oauthConf = &oauth2.Config{ + Scopes: []string{ + "wl.signin", // Allow single sign-on capabilities + "wl.offline_access", // Allow receiving a refresh token + "onedrive.readwrite", // r/w perms to all of a user's OneDrive files + }, + Endpoint: oauth2.Endpoint{ + AuthURL: "https://login.live.com/oauth20_authorize.srf", + TokenURL: "https://login.live.com/oauth20_token.srf", + }, +} + +// Items represents a collection of Items +type items struct { + Collection []*item `json:"value"` + NextLink string `json:"@odata.nextLink"` +} + +// The Item resource type represents metadata for an item in OneDrive. +// Since we only need the name and the size, this structure incomplete. +// see: https://dev.onedrive.com/resources/item.htm +type item struct { + Name string `json:"name"` + Size int64 `json:"size"` +} + +// Action is used for the paramters of an onedrive request +type action string + +const ( + list action = ":/children?select=name" + content = ":/content" + none = "" +) + +func (be *onedrive) odPath(t backend.Type, name string, a action) string { + if t == backend.Config || name == "" { + return onedriveBase + string(t) + string(a) + } + return onedriveBase + string(t) + "/" + name + string(a) +} + +// Open opens the onedrive backend. +func Open(cfg Config) (backend.Backend, error) { + debug.Log("onedrive.Open", "open, config %#v", cfg) + + oauthConf.ClientID = cfg.ClientID + oauthConf.ClientSecret = cfg.ClientSecret + + client := oauthConf.Client(oauth2.NoContext, &cfg.Token) + + be := &onedrive{client: client} + be.createConnections() + + return be, nil +} + +func (be *onedrive) createConnections() { + be.connChan = make(chan struct{}, connLimit) + for i := 0; i < connLimit; i++ { + be.connChan <- struct{}{} + } +} + +// List returns a channel that yields all names of blobs of type t. A +// goroutine is started for this. If the channel done is closed, sending +// stops. +func (be *onedrive) List(t backend.Type, done <-chan struct{}) <-chan string { + debug.Log("onedrive.List", "listing %v", t) + ch := make(chan string) + + collection := []*item{} + + path := be.odPath(t, "", list) + + for path != "" { + debug.Log("onedrive.List", "path %v", path) + resp, err := be.client.Get(path) + if resp != nil { + defer resp.Body.Close() + } + if err != nil { + close(ch) + return ch + } + + respData, err := ioutil.ReadAll(resp.Body) + if err != nil { + close(ch) + return ch + } + + var jsonData items + err = json.Unmarshal([]byte(respData), &jsonData) + if err != nil { + close(ch) + return ch + } + + collection = append(collection, jsonData.Collection...) + + // If a collection exceeds the default page size (200 items), + // the @odata.nextLink property is returned and we have to fetch the + // next page. + path = jsonData.NextLink + } + + go func() { + defer close(ch) + for _, obj := range collection { + m := obj.Name + + if m == "" { + continue + } + + select { + case ch <- m: + case <-done: + return + } + } + }() + + return ch +} + +// Load returns the data stored in the backend for h at the given offset +// and saves it in p. Load has the same semantics as io.ReaderAt. +func (be *onedrive) Load(h backend.Handle, p []byte, off int64) (n int, err error) { + debug.Log("onedrive.Load", "load offset %v length %v", off, len(p)) + path := be.odPath(h.Type, h.Name, content) + + req, err := http.NewRequest("GET", path, nil) + if err != nil { + debug.Log("onedrive.Load", " err %v", err) + return 0, err + } + + transport := be.client.Transport + resp, err := transport.RoundTrip(req) + if err != nil { + debug.Log("onedrive.Load", " err %v", err) + return 0, err + } + if resp.StatusCode != 302 { + debug.Log("onedrive.Load()", "no redirect - resp %v", resp) + return 0, errors.New("no redirect to content location") + } + + req, err = http.NewRequest("GET", resp.Header.Get("Location"), nil) + if err != nil { + debug.Log("onedrive.Load", " err %v", err) + return 0, err + } + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p)-1))) + + resp, err = be.client.Do(req) + if err != nil { + debug.Log("onedrive.Load", " err %v", err) + return 0, err + } + + <-be.connChan + defer func() { + be.connChan <- struct{}{} + }() + + n, err = io.ReadFull(resp.Body, p) + if err != nil { + debug.Log("onedrive.Load", "GetObject() err %v", err) + return 0, err + } + + return n, nil +} + +// Save stores data in the backend at the handle. +func (be onedrive) Save(h backend.Handle, p []byte) (err error) { + debug.Log("onedrive.Save", "name %v", h.Name) + path := be.odPath(h.Type, h.Name, content) + + // Check file does not already exist + resp, err := be.client.Get(path) + if err == nil && resp.StatusCode == 200 { + debug.Log("onedrive.Save", "%v already exists", h) + return errors.New("key already exists") + } + + <-be.connChan + defer func() { + be.connChan <- struct{}{} + }() + + debug.Log("onedrive.Save", "PutObject(%v, %v)", + path, int64(len(p))) + + req, err := http.NewRequest("PUT", path, bytes.NewReader(p)) + if err != nil { + debug.Log("onedrive.Save", " err %v", err) + return err + } + + resp, err = be.client.Do(req) + if err != nil { + debug.Log("onedrive.Save", " err %v ", err) + return err + } + + if resp.StatusCode >= 400 { + debug.Log("onedrive.Save", " resp %v ", resp) + return errors.New("Invalid response code") + } + + debug.Log("onedrive.Save", "%v -> %v bytes, err %#v", path, resp.ContentLength, err) + + return err +} + +// Stat returns information about a blob. +func (be *onedrive) Stat(h backend.Handle) (backend.BlobInfo, error) { + debug.Log("onedrive.Stat", "name %v", h.Name) + path := be.odPath(h.Type, h.Name, none) + + resp, err := be.client.Get(path) + if err != nil { + debug.Log("onedrive.Stat", "GetObject() err %v", err) + return backend.BlobInfo{}, err + } + + respData, err := ioutil.ReadAll(resp.Body) + if err != nil { + debug.Log("onedrive.Stat", "GetObject() err %v", err) + return backend.BlobInfo{}, err + } + + var jsonData item + err = json.Unmarshal([]byte(respData), &jsonData) + if err != nil { + debug.Log("onedrive.Stat", "GetObject() err %v", err) + return backend.BlobInfo{}, err + } + + return backend.BlobInfo{Size: jsonData.Size}, nil +} + +// Test returns true if a blob of the given type and name exists in the backend. +func (be *onedrive) Test(t backend.Type, name string) (bool, error) { + debug.Log("onedrive.Test", "name %v", name) + found := false + path := be.odPath(t, name, none) + resp, err := be.client.Get(path) + if err == nil && resp.StatusCode == 200 { + found = true + } + + // If error, then not found + return found, nil +} + +// Remove removes the blob with the given name and type. +func (be *onedrive) Remove(t backend.Type, name string) error { + debug.Log("onedrive.Remove", "name %v", name) + path := be.odPath(t, name, none) + req, err := http.NewRequest("DELETE", path, nil) + if err != nil { + debug.Log("onedrive.Remove", " err %v", err) + return err + } + + _, err = be.client.Do(req) + if err != nil { + debug.Log("onedrive.Remove", " err %v", err) + return err + } + return err +} + +// Location returns this backend's location (the bucket name). +// TODO: return path? +func (be *onedrive) Location() string { + return "restic" +} + +// Close does nothing +func (be *onedrive) Close() error { return nil } diff --git a/src/restic/location/location.go b/src/restic/location/location.go index 23e0af37b..16e0939ee 100644 --- a/src/restic/location/location.go +++ b/src/restic/location/location.go @@ -5,6 +5,7 @@ import ( "strings" "restic/backend/local" + "restic/backend/onedrive" "restic/backend/rest" "restic/backend/s3" "restic/backend/sftp" @@ -29,6 +30,7 @@ var parsers = []parser{ {"sftp", sftp.ParseConfig}, {"s3", s3.ParseConfig}, {"rest", rest.ParseConfig}, + {"onedrive", onedrive.ParseConfig}, } // Parse extracts repository location information from the string s. If s