mirror of
https://github.com/restic/restic.git
synced 2025-03-16 00:00:05 +01:00
first working draft of onedrive backend
This commit is contained in:
parent
4a1cf76ae1
commit
7186236e46
4 changed files with 365 additions and 0 deletions
|
@ -10,6 +10,7 @@ import (
|
|||
|
||||
"restic/backend"
|
||||
"restic/backend/local"
|
||||
"restic/backend/onedrive"
|
||||
"restic/backend/rest"
|
||||
"restic/backend/s3"
|
||||
"restic/backend/sftp"
|
||||
|
@ -251,6 +252,9 @@ func open(s string) (backend.Backend, error) {
|
|||
return s3.Open(cfg)
|
||||
case "rest":
|
||||
return rest.Open(loc.Config.(rest.Config))
|
||||
case "onedrive":
|
||||
debug.Log("open", "opening ondrive repository")
|
||||
return onedrive.Open(loc.Config.(onedrive.Config))
|
||||
}
|
||||
|
||||
debug.Log("open", "invalid repository location: %v", s)
|
||||
|
@ -286,6 +290,9 @@ func create(s string) (backend.Backend, error) {
|
|||
return s3.Open(cfg)
|
||||
case "rest":
|
||||
return rest.Open(loc.Config.(rest.Config))
|
||||
case "onedrive":
|
||||
debug.Log("open", "opening ondrive repository")
|
||||
return onedrive.Open(loc.Config.(onedrive.Config))
|
||||
}
|
||||
|
||||
debug.Log("open", "invalid repository scheme: %v", s)
|
||||
|
|
45
src/restic/backend/onedrive/config.go
Normal file
45
src/restic/backend/onedrive/config.go
Normal file
|
@ -0,0 +1,45 @@
|
|||
package onedrive
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
|
||||
// Config contains all configuration necessary to connect to onedrive.
|
||||
type Config struct {
|
||||
Token oauth2.Token `json:"token"`
|
||||
ClientID string `json:"client_id"`
|
||||
ClientSecret string `json:"client_secret"`
|
||||
}
|
||||
|
||||
// ParseConfig parses the string s and extracts the onedrive config. The
|
||||
// supported configuration format is onedrive:configfile.json
|
||||
func ParseConfig(s string) (interface{}, error) {
|
||||
if strings.HasPrefix(s, "onedrive:") {
|
||||
s = s[9:]
|
||||
|
||||
data := strings.SplitN(s, ":", 1)
|
||||
if len(data) != 1 {
|
||||
return nil, errors.New("onedrive: invalid format")
|
||||
}
|
||||
|
||||
file, err := ioutil.ReadFile(data[0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("onedrive: reading tokenfile (%s) failed with '%s'", data[0], err)
|
||||
}
|
||||
|
||||
var cfg Config
|
||||
if err := json.Unmarshal(file, &cfg); err != nil {
|
||||
return nil, fmt.Errorf("onedrive: token de-serialization failed with '%s'", err)
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
return nil, errors.New("onedrive: invalid format")
|
||||
}
|
311
src/restic/backend/onedrive/onedrive.go
Normal file
311
src/restic/backend/onedrive/onedrive.go
Normal file
|
@ -0,0 +1,311 @@
|
|||
package onedrive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
|
||||
"golang.org/x/oauth2"
|
||||
|
||||
"restic/backend"
|
||||
"restic/debug"
|
||||
)
|
||||
|
||||
const connLimit = 10
|
||||
const onedriveBase = "https://api.onedrive.com/v1.0/drive/special/approot:/"
|
||||
|
||||
type onedrive struct {
|
||||
client *http.Client
|
||||
connChan chan struct{}
|
||||
}
|
||||
|
||||
var oauthConf = &oauth2.Config{
|
||||
Scopes: []string{
|
||||
"wl.signin", // Allow single sign-on capabilities
|
||||
"wl.offline_access", // Allow receiving a refresh token
|
||||
"onedrive.readwrite", // r/w perms to all of a user's OneDrive files
|
||||
},
|
||||
Endpoint: oauth2.Endpoint{
|
||||
AuthURL: "https://login.live.com/oauth20_authorize.srf",
|
||||
TokenURL: "https://login.live.com/oauth20_token.srf",
|
||||
},
|
||||
}
|
||||
|
||||
// Items represents a collection of Items
|
||||
type items struct {
|
||||
Collection []*item `json:"value"`
|
||||
NextLink string `json:"@odata.nextLink"`
|
||||
}
|
||||
|
||||
// The Item resource type represents metadata for an item in OneDrive.
|
||||
// Since we only need the name and the size, this structure incomplete.
|
||||
// see: https://dev.onedrive.com/resources/item.htm
|
||||
type item struct {
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
// Action is used for the paramters of an onedrive request
|
||||
type action string
|
||||
|
||||
const (
|
||||
list action = ":/children?select=name"
|
||||
content = ":/content"
|
||||
none = ""
|
||||
)
|
||||
|
||||
func (be *onedrive) odPath(t backend.Type, name string, a action) string {
|
||||
if t == backend.Config || name == "" {
|
||||
return onedriveBase + string(t) + string(a)
|
||||
}
|
||||
return onedriveBase + string(t) + "/" + name + string(a)
|
||||
}
|
||||
|
||||
// Open opens the onedrive backend.
|
||||
func Open(cfg Config) (backend.Backend, error) {
|
||||
debug.Log("onedrive.Open", "open, config %#v", cfg)
|
||||
|
||||
oauthConf.ClientID = cfg.ClientID
|
||||
oauthConf.ClientSecret = cfg.ClientSecret
|
||||
|
||||
client := oauthConf.Client(oauth2.NoContext, &cfg.Token)
|
||||
|
||||
be := &onedrive{client: client}
|
||||
be.createConnections()
|
||||
|
||||
return be, nil
|
||||
}
|
||||
|
||||
func (be *onedrive) createConnections() {
|
||||
be.connChan = make(chan struct{}, connLimit)
|
||||
for i := 0; i < connLimit; i++ {
|
||||
be.connChan <- struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// List returns a channel that yields all names of blobs of type t. A
|
||||
// goroutine is started for this. If the channel done is closed, sending
|
||||
// stops.
|
||||
func (be *onedrive) List(t backend.Type, done <-chan struct{}) <-chan string {
|
||||
debug.Log("onedrive.List", "listing %v", t)
|
||||
ch := make(chan string)
|
||||
|
||||
collection := []*item{}
|
||||
|
||||
path := be.odPath(t, "", list)
|
||||
|
||||
for path != "" {
|
||||
debug.Log("onedrive.List", "path %v", path)
|
||||
resp, err := be.client.Get(path)
|
||||
if resp != nil {
|
||||
defer resp.Body.Close()
|
||||
}
|
||||
if err != nil {
|
||||
close(ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
respData, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
close(ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
var jsonData items
|
||||
err = json.Unmarshal([]byte(respData), &jsonData)
|
||||
if err != nil {
|
||||
close(ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
collection = append(collection, jsonData.Collection...)
|
||||
|
||||
// If a collection exceeds the default page size (200 items),
|
||||
// the @odata.nextLink property is returned and we have to fetch the
|
||||
// next page.
|
||||
path = jsonData.NextLink
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
for _, obj := range collection {
|
||||
m := obj.Name
|
||||
|
||||
if m == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case ch <- m:
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
// Load returns the data stored in the backend for h at the given offset
|
||||
// and saves it in p. Load has the same semantics as io.ReaderAt.
|
||||
func (be *onedrive) Load(h backend.Handle, p []byte, off int64) (n int, err error) {
|
||||
debug.Log("onedrive.Load", "load offset %v length %v", off, len(p))
|
||||
path := be.odPath(h.Type, h.Name, content)
|
||||
|
||||
req, err := http.NewRequest("GET", path, nil)
|
||||
if err != nil {
|
||||
debug.Log("onedrive.Load", " err %v", err)
|
||||
return 0, err
|
||||
}
|
||||
|
||||
transport := be.client.Transport
|
||||
resp, err := transport.RoundTrip(req)
|
||||
if err != nil {
|
||||
debug.Log("onedrive.Load", " err %v", err)
|
||||
return 0, err
|
||||
}
|
||||
if resp.StatusCode != 302 {
|
||||
debug.Log("onedrive.Load()", "no redirect - resp %v", resp)
|
||||
return 0, errors.New("no redirect to content location")
|
||||
}
|
||||
|
||||
req, err = http.NewRequest("GET", resp.Header.Get("Location"), nil)
|
||||
if err != nil {
|
||||
debug.Log("onedrive.Load", " err %v", err)
|
||||
return 0, err
|
||||
}
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p)-1)))
|
||||
|
||||
resp, err = be.client.Do(req)
|
||||
if err != nil {
|
||||
debug.Log("onedrive.Load", " err %v", err)
|
||||
return 0, err
|
||||
}
|
||||
|
||||
<-be.connChan
|
||||
defer func() {
|
||||
be.connChan <- struct{}{}
|
||||
}()
|
||||
|
||||
n, err = io.ReadFull(resp.Body, p)
|
||||
if err != nil {
|
||||
debug.Log("onedrive.Load", "GetObject() err %v", err)
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// Save stores data in the backend at the handle.
|
||||
func (be onedrive) Save(h backend.Handle, p []byte) (err error) {
|
||||
debug.Log("onedrive.Save", "name %v", h.Name)
|
||||
path := be.odPath(h.Type, h.Name, content)
|
||||
|
||||
// Check file does not already exist
|
||||
resp, err := be.client.Get(path)
|
||||
if err == nil && resp.StatusCode == 200 {
|
||||
debug.Log("onedrive.Save", "%v already exists", h)
|
||||
return errors.New("key already exists")
|
||||
}
|
||||
|
||||
<-be.connChan
|
||||
defer func() {
|
||||
be.connChan <- struct{}{}
|
||||
}()
|
||||
|
||||
debug.Log("onedrive.Save", "PutObject(%v, %v)",
|
||||
path, int64(len(p)))
|
||||
|
||||
req, err := http.NewRequest("PUT", path, bytes.NewReader(p))
|
||||
if err != nil {
|
||||
debug.Log("onedrive.Save", " err %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err = be.client.Do(req)
|
||||
if err != nil {
|
||||
debug.Log("onedrive.Save", " err %v ", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
debug.Log("onedrive.Save", " resp %v ", resp)
|
||||
return errors.New("Invalid response code")
|
||||
}
|
||||
|
||||
debug.Log("onedrive.Save", "%v -> %v bytes, err %#v", path, resp.ContentLength, err)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Stat returns information about a blob.
|
||||
func (be *onedrive) Stat(h backend.Handle) (backend.BlobInfo, error) {
|
||||
debug.Log("onedrive.Stat", "name %v", h.Name)
|
||||
path := be.odPath(h.Type, h.Name, none)
|
||||
|
||||
resp, err := be.client.Get(path)
|
||||
if err != nil {
|
||||
debug.Log("onedrive.Stat", "GetObject() err %v", err)
|
||||
return backend.BlobInfo{}, err
|
||||
}
|
||||
|
||||
respData, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
debug.Log("onedrive.Stat", "GetObject() err %v", err)
|
||||
return backend.BlobInfo{}, err
|
||||
}
|
||||
|
||||
var jsonData item
|
||||
err = json.Unmarshal([]byte(respData), &jsonData)
|
||||
if err != nil {
|
||||
debug.Log("onedrive.Stat", "GetObject() err %v", err)
|
||||
return backend.BlobInfo{}, err
|
||||
}
|
||||
|
||||
return backend.BlobInfo{Size: jsonData.Size}, nil
|
||||
}
|
||||
|
||||
// Test returns true if a blob of the given type and name exists in the backend.
|
||||
func (be *onedrive) Test(t backend.Type, name string) (bool, error) {
|
||||
debug.Log("onedrive.Test", "name %v", name)
|
||||
found := false
|
||||
path := be.odPath(t, name, none)
|
||||
resp, err := be.client.Get(path)
|
||||
if err == nil && resp.StatusCode == 200 {
|
||||
found = true
|
||||
}
|
||||
|
||||
// If error, then not found
|
||||
return found, nil
|
||||
}
|
||||
|
||||
// Remove removes the blob with the given name and type.
|
||||
func (be *onedrive) Remove(t backend.Type, name string) error {
|
||||
debug.Log("onedrive.Remove", "name %v", name)
|
||||
path := be.odPath(t, name, none)
|
||||
req, err := http.NewRequest("DELETE", path, nil)
|
||||
if err != nil {
|
||||
debug.Log("onedrive.Remove", " err %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = be.client.Do(req)
|
||||
if err != nil {
|
||||
debug.Log("onedrive.Remove", " err %v", err)
|
||||
return err
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Location returns this backend's location (the bucket name).
|
||||
// TODO: return path?
|
||||
func (be *onedrive) Location() string {
|
||||
return "restic"
|
||||
}
|
||||
|
||||
// Close does nothing
|
||||
func (be *onedrive) Close() error { return nil }
|
|
@ -5,6 +5,7 @@ import (
|
|||
"strings"
|
||||
|
||||
"restic/backend/local"
|
||||
"restic/backend/onedrive"
|
||||
"restic/backend/rest"
|
||||
"restic/backend/s3"
|
||||
"restic/backend/sftp"
|
||||
|
@ -29,6 +30,7 @@ var parsers = []parser{
|
|||
{"sftp", sftp.ParseConfig},
|
||||
{"s3", s3.ParseConfig},
|
||||
{"rest", rest.ParseConfig},
|
||||
{"onedrive", onedrive.ParseConfig},
|
||||
}
|
||||
|
||||
// Parse extracts repository location information from the string s. If s
|
||||
|
|
Loading…
Add table
Reference in a new issue