diff --git a/.travis.yml b/.travis.yml index f2d6b6bb..57042317 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: go sudo: false go: - - 1.6.4 - 1.7.5 - 1.8 - tip @@ -17,8 +16,6 @@ env: matrix: exclude: - - os: osx - go: 1.6.4 - os: osx go: 1.7.5 - os: osx diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5529ea7f..5ffdc40d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -77,7 +77,7 @@ Just clone the repository, `cd` to it and run `gb build` to build the binary: [...] $ bin/restic version restic compiled manually - compiled at unknown time with go1.6 + compiled at unknown time with go1.7 The following commands can be used to run all the tests: diff --git a/README.md b/README.md index cc90f7b6..3fe4c0ed 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ You can download the latest pre-compiled binary from the [restic release page](h Build restic ============ -Install Go/Golang (at least version 1.6), then run `go run build.go`, +Install Go/Golang (at least version 1.7), then run `go run build.go`, afterwards you'll find the binary in the current directory: $ go run build.go diff --git a/Vagrantfile b/Vagrantfile index a26aa6b6..ee872b3f 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,7 +1,7 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -GO_VERSION = '1.6' +GO_VERSION = '1.7' def packages_freebsd return <<-EOF diff --git a/doc/Manual.md b/doc/Manual.md index 513ef612..5f2a470f 100644 --- a/doc/Manual.md +++ b/doc/Manual.md @@ -27,7 +27,7 @@ $ pacaur -S restic-git # Building restic -restic is written in the Go programming language and you need at least Go version 1.6. +restic is written in the Go programming language and you need at least Go version 1.7. Building restic may also work with older versions of Go, but that's not supported. See the [Getting started](https://golang.org/doc/install) guide of the Go project for instructions how to install Go. diff --git a/src/cmds/restic/cmd_backup.go b/src/cmds/restic/cmd_backup.go index ee4a3e61..d98e7e56 100644 --- a/src/cmds/restic/cmd_backup.go +++ b/src/cmds/restic/cmd_backup.go @@ -67,12 +67,12 @@ func init() { f := cmdBackup.Flags() f.StringVar(&backupOptions.Parent, "parent", "", "use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)") f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`) - f.StringSliceVarP(&backupOptions.Excludes, "exclude", "e", []string{}, "exclude a `pattern` (can be specified multiple times)") + f.StringSliceVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)") f.StringVar(&backupOptions.ExcludeFile, "exclude-file", "", "read exclude patterns from a file") f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems") f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin") f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "file name to use when reading from stdin") - f.StringSliceVar(&backupOptions.Tags, "tag", []string{}, "add a `tag` for the new snapshot (can be specified multiple times)") + f.StringSliceVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)") f.StringVar(&backupOptions.Hostname, "hostname", hostname, "set the `hostname` for the snapshot manually") f.StringVar(&backupOptions.FilesFrom, "files-from", "", "read the files to backup from file (can be combined with file args)") } @@ -391,7 +391,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error { // Find last snapshot to set it as parent, if not already set if !opts.Force && parentSnapshotID == nil { - id, err := restic.FindLatestSnapshot(repo, target, opts.Hostname) + id, err := restic.FindLatestSnapshot(repo, target, opts.Tags, opts.Hostname) if err == nil { parentSnapshotID = &id } else if err != restic.ErrNoSnapshotFound { diff --git a/src/cmds/restic/cmd_check.go b/src/cmds/restic/cmd_check.go index cd041a42..2f0064f1 100644 --- a/src/cmds/restic/cmd_check.go +++ b/src/cmds/restic/cmd_check.go @@ -24,7 +24,7 @@ finds. It can also be used to read all data and therefore simulate a restore. }, } -// CheckOptions bundle all options for the 'check' command. +// CheckOptions bundles all options for the 'check' command. type CheckOptions struct { ReadData bool CheckUnused bool diff --git a/src/cmds/restic/cmd_find.go b/src/cmds/restic/cmd_find.go index cb9e6f59..23c39485 100644 --- a/src/cmds/restic/cmd_find.go +++ b/src/cmds/restic/cmd_find.go @@ -1,6 +1,7 @@ package main import ( + "context" "path/filepath" "strings" "time" @@ -24,12 +25,16 @@ repo. `, }, } -// FindOptions bundle all options for the find command. +// FindOptions bundles all options for the find command. type FindOptions struct { Oldest string Newest string - Snapshot string + Snapshots []string CaseInsensitive bool + ListLong bool + Host string + Paths []string + Tags []string } var findOptions FindOptions @@ -40,8 +45,13 @@ func init() { f := cmdFind.Flags() f.StringVarP(&findOptions.Oldest, "oldest", "o", "", "oldest modification date/time") f.StringVarP(&findOptions.Newest, "newest", "n", "", "newest modification date/time") - f.StringVarP(&findOptions.Snapshot, "snapshot", "s", "", "snapshot ID to search in") + f.StringSliceVarP(&findOptions.Snapshots, "snapshot", "s", nil, "snapshot `id` to search in (can be given multiple times)") f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern") + f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode") + + f.StringVarP(&findOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given") + f.StringSliceVar(&findOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given") + f.StringSliceVar(&findOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given") } type findPattern struct { @@ -50,11 +60,6 @@ type findPattern struct { ignoreCase bool } -type findResult struct { - node *restic.Node - path string -} - var timeFormats = []string{ "2006-01-02", "2006-01-02 15:04", @@ -79,14 +84,14 @@ func parseTime(str string) (time.Time, error) { return time.Time{}, errors.Fatalf("unable to parse time: %q", str) } -func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, path string) ([]findResult, error) { +func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, prefix string, snapshotID *string) error { debug.Log("checking tree %v\n", id) + tree, err := repo.LoadTree(id) if err != nil { - return nil, err + return err } - results := []findResult{} for _, node := range tree.Nodes { debug.Log(" testing entry %q\n", node.Name) @@ -97,7 +102,7 @@ func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, path m, err := filepath.Match(pat.pattern, name) if err != nil { - return nil, err + return err } if m { @@ -112,46 +117,32 @@ func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, path continue } - results = append(results, findResult{node: node, path: path}) + if snapshotID != nil { + Verbosef("Found matching entries in snapshot %s\n", *snapshotID) + snapshotID = nil + } + Printf(formatNode(prefix, node, findOptions.ListLong) + "\n") } else { debug.Log(" pattern does not match\n") } if node.Type == "dir" { - subdirResults, err := findInTree(repo, pat, *node.Subtree, filepath.Join(path, node.Name)) - if err != nil { - return nil, err + if err := findInTree(repo, pat, *node.Subtree, filepath.Join(prefix, node.Name), snapshotID); err != nil { + return err } - - results = append(results, subdirResults...) } } - return results, nil + return nil } -func findInSnapshot(repo *repository.Repository, pat findPattern, id restic.ID) error { - debug.Log("searching in snapshot %s\n for entries within [%s %s]", id.Str(), pat.oldest, pat.newest) +func findInSnapshot(repo *repository.Repository, sn *restic.Snapshot, pat findPattern) error { + debug.Log("searching in snapshot %s\n for entries within [%s %s]", sn.ID(), pat.oldest, pat.newest) - sn, err := restic.LoadSnapshot(repo, id) - if err != nil { + snapshotID := sn.ID().Str() + if err := findInTree(repo, pat, *sn.Tree, string(filepath.Separator), &snapshotID); err != nil { return err } - - results, err := findInTree(repo, pat, *sn.Tree, string(filepath.Separator)) - if err != nil { - return err - } - - if len(results) == 0 { - return nil - } - Verbosef("found %d matching entries in snapshot %s\n", len(results), id) - for _, res := range results { - res.node.Name = filepath.Join(res.path, res.node.Name) - Printf(" %s\n", res.node) - } - return nil } @@ -160,21 +151,21 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error { return errors.Fatal("wrong number of arguments") } - var ( - err error - pat findPattern - ) + var err error + pat := findPattern{pattern: args[0]} + if opts.CaseInsensitive { + pat.pattern = strings.ToLower(pat.pattern) + pat.ignoreCase = true + } if opts.Oldest != "" { - pat.oldest, err = parseTime(opts.Oldest) - if err != nil { + if pat.oldest, err = parseTime(opts.Oldest); err != nil { return err } } if opts.Newest != "" { - pat.newest, err = parseTime(opts.Newest) - if err != nil { + if pat.newest, err = parseTime(opts.Newest); err != nil { return err } } @@ -192,33 +183,14 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error { } } - err = repo.LoadIndex() - if err != nil { + if err = repo.LoadIndex(); err != nil { return err } - pat.pattern = args[0] - - if opts.CaseInsensitive { - pat.pattern = strings.ToLower(pat.pattern) - pat.ignoreCase = true - } - - if opts.Snapshot != "" { - snapshotID, err := restic.FindSnapshot(repo, opts.Snapshot) - if err != nil { - return errors.Fatalf("invalid id %q: %v", args[1], err) - } - - return findInSnapshot(repo, pat, snapshotID) - } - - done := make(chan struct{}) - defer close(done) - for snapshotID := range repo.List(restic.SnapshotFile, done) { - err := findInSnapshot(repo, pat, snapshotID) - - if err != nil { + ctx, cancel := context.WithCancel(gopts.ctx) + defer cancel() + for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, opts.Snapshots) { + if err = findInSnapshot(repo, sn, pat); err != nil { return err } } diff --git a/src/cmds/restic/cmd_forget.go b/src/cmds/restic/cmd_forget.go index 7c762728..bc372fc5 100644 --- a/src/cmds/restic/cmd_forget.go +++ b/src/cmds/restic/cmd_forget.go @@ -1,14 +1,12 @@ package main import ( - "encoding/hex" + "context" "encoding/json" "restic" "sort" "strings" - "restic/errors" - "github.com/spf13/cobra" ) @@ -27,13 +25,12 @@ data after 'forget' was run successfully, see the 'prune' command. `, // ForgetOptions collects all options for the forget command. type ForgetOptions struct { - Last int - Hourly int - Daily int - Weekly int - Monthly int - Yearly int - + Last int + Hourly int + Daily int + Weekly int + Monthly int + Yearly int KeepTags []string Host string @@ -83,32 +80,43 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error { return err } - // Process all snapshot IDs given as arguments. - if len(args) != 0 { - for _, s := range args { - // Parse argument as hex string. - if _, err := hex.DecodeString(s); err != nil { - Warnf("argument %q is not a snapshot ID, ignoring\n", s) - continue - } - id, err := restic.FindSnapshot(repo, s) - if err != nil { - Warnf("could not find a snapshot for ID %q, ignoring\n", s) - continue - } + // group by hostname and dirs + type key struct { + Hostname string + Paths []string + Tags []string + } + snapshotGroups := make(map[string]restic.Snapshots) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) { + if len(args) > 0 { + // When explicit snapshots args are given, remove them immediately. if !opts.DryRun { - h := restic.Handle{Type: restic.SnapshotFile, Name: id.String()} - err = repo.Backend().Remove(h) - if err != nil { + h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} + if err = repo.Backend().Remove(h); err != nil { return err } - - Verbosef("removed snapshot %v\n", id.Str()) + Verbosef("removed snapshot %v\n", sn.ID().Str()) } else { - Verbosef("would remove snapshot %v\n", id.Str()) + Verbosef("would have removed snapshot %v\n", sn.ID().Str()) } + } else { + var tags []string + if opts.GroupByTags { + tags = sn.Tags + sort.StringSlice(tags).Sort() + } + sort.StringSlice(sn.Paths).Sort() + k, err := json.Marshal(key{Hostname: sn.Hostname, Tags: tags, Paths: sn.Paths}) + if err != nil { + return err + } + snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn) } + } + if len(args) > 0 { return nil } @@ -122,53 +130,17 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error { Tags: opts.KeepTags, } - snapshots, err := restic.LoadAllSnapshots(repo) - if err != nil { - return err - } - - // Group snapshots by hostname and dirs. - type key struct { - Hostname string - Paths []string - Tags []string - } - - snapshotGroups := make(map[string]restic.Snapshots) - - for _, sn := range snapshots { - if opts.Host != "" && sn.Hostname != opts.Host { - continue - } - - if !sn.HasTags(opts.Tags) { - continue - } - - if !sn.HasPaths(opts.Paths) { - continue - } - - var tags []string - if opts.GroupByTags { - sort.StringSlice(sn.Tags).Sort() - tags = sn.Tags - } - sort.StringSlice(sn.Paths).Sort() - k, _ := json.Marshal(key{Hostname: sn.Hostname, Tags: tags, Paths: sn.Paths}) - snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn) - } - if len(snapshotGroups) == 0 { - return errors.Fatal("no snapshots remained after filtering") - } if policy.Empty() { Verbosef("no policy was specified, no snapshots will be removed\n") + return nil } removeSnapshots := 0 for k, snapshotGroup := range snapshotGroups { var key key - json.Unmarshal([]byte(k), &key) + if json.Unmarshal([]byte(k), &key) != nil { + return err + } if opts.GroupByTags { Printf("snapshots for host %v, tags [%v], paths: [%v]:\n\n", key.Hostname, strings.Join(key.Tags, ", "), strings.Join(key.Paths, ", ")) } else { diff --git a/src/cmds/restic/cmd_key.go b/src/cmds/restic/cmd_key.go index 7e51def4..052dd5b8 100644 --- a/src/cmds/restic/cmd_key.go +++ b/src/cmds/restic/cmd_key.go @@ -1,13 +1,13 @@ package main import ( + "context" "fmt" "restic" - - "github.com/spf13/cobra" - "restic/errors" "restic/repository" + + "github.com/spf13/cobra" ) var cmdKey = &cobra.Command{ @@ -25,15 +25,12 @@ func init() { cmdRoot.AddCommand(cmdKey) } -func listKeys(s *repository.Repository) error { +func listKeys(ctx context.Context, s *repository.Repository) error { tab := NewTable() tab.Header = fmt.Sprintf(" %-10s %-10s %-10s %s", "ID", "User", "Host", "Created") tab.RowFormat = "%s%-10s %-10s %-10s %s" - done := make(chan struct{}) - defer close(done) - - for id := range s.List(restic.KeyFile, done) { + for id := range s.List(restic.KeyFile, ctx.Done()) { k, err := repository.LoadKey(s, id.String()) if err != nil { Warnf("LoadKey() failed: %v\n", err) @@ -124,6 +121,9 @@ func runKey(gopts GlobalOptions, args []string) error { return errors.Fatal("wrong number of arguments") } + ctx, cancel := context.WithCancel(gopts.ctx) + defer cancel() + repo, err := OpenRepository(gopts) if err != nil { return err @@ -137,7 +137,7 @@ func runKey(gopts GlobalOptions, args []string) error { return err } - return listKeys(repo) + return listKeys(ctx, repo) case "add": lock, err := lockRepo(repo) defer unlockRepo(lock) diff --git a/src/cmds/restic/cmd_ls.go b/src/cmds/restic/cmd_ls.go index c6c05bec..7d613b45 100644 --- a/src/cmds/restic/cmd_ls.go +++ b/src/cmds/restic/cmd_ls.go @@ -1,8 +1,7 @@ package main import ( - "fmt" - "os" + "context" "path/filepath" "github.com/spf13/cobra" @@ -13,7 +12,7 @@ import ( ) var cmdLs = &cobra.Command{ - Use: "ls [flags] snapshot-ID", + Use: "ls [flags] [snapshot-ID ...]", Short: "list files in a snapshot", Long: ` The "ls" command allows listing files and directories in a snapshot. @@ -21,7 +20,7 @@ The "ls" command allows listing files and directories in a snapshot. The special snapshot-ID "latest" can be used to list files and directories of the latest snapshot in the repository. `, RunE: func(cmd *cobra.Command, args []string) error { - return runLs(globalOptions, args) + return runLs(lsOptions, globalOptions, args) }, } @@ -29,6 +28,7 @@ The special snapshot-ID "latest" can be used to list files and directories of th type LsOptions struct { ListLong bool Host string + Tags []string Paths []string } @@ -40,42 +40,22 @@ func init() { flags := cmdLs.Flags() flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode") - flags.StringVarP(&lsOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`) - flags.StringSliceVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"") + flags.StringVarP(&lsOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given") + flags.StringSliceVar(&lsOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot ID is given") + flags.StringSliceVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given") } -func printNode(prefix string, n *restic.Node) string { - if !lsOptions.ListLong { - return filepath.Join(prefix, n.Name) - } - - switch n.Type { - case "file": - return fmt.Sprintf("%s %5d %5d %6d %s %s", - n.Mode, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name)) - case "dir": - return fmt.Sprintf("%s %5d %5d %6d %s %s", - n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name)) - case "symlink": - return fmt.Sprintf("%s %5d %5d %6d %s %s -> %s", - n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name), n.LinkTarget) - default: - return fmt.Sprintf("", n.Type, n.Name) - } -} - -func printTree(prefix string, repo *repository.Repository, id restic.ID) error { - tree, err := repo.LoadTree(id) +func printTree(repo *repository.Repository, id *restic.ID, prefix string) error { + tree, err := repo.LoadTree(*id) if err != nil { return err } for _, entry := range tree.Nodes { - Printf(printNode(prefix, entry) + "\n") + Printf(formatNode(prefix, entry, lsOptions.ListLong) + "\n") if entry.Type == "dir" && entry.Subtree != nil { - err = printTree(filepath.Join(prefix, entry.Name), repo, *entry.Subtree) - if err != nil { + if err = printTree(repo, entry.Subtree, filepath.Join(prefix, entry.Name)); err != nil { return err } } @@ -84,9 +64,9 @@ func printTree(prefix string, repo *repository.Repository, id restic.ID) error { return nil } -func runLs(gopts GlobalOptions, args []string) error { - if len(args) < 1 || len(args) > 2 { - return errors.Fatal("no snapshot ID given") +func runLs(opts LsOptions, gopts GlobalOptions, args []string) error { + if len(args) == 0 && opts.Host == "" && len(opts.Tags) == 0 && len(opts.Paths) == 0 { + return errors.Fatal("Invalid arguments, either give one or more snapshot IDs or set filters.") } repo, err := OpenRepository(gopts) @@ -94,32 +74,18 @@ func runLs(gopts GlobalOptions, args []string) error { return err } - err = repo.LoadIndex() - if err != nil { + if err = repo.LoadIndex(); err != nil { return err } - snapshotIDString := args[0] - var id restic.ID + ctx, cancel := context.WithCancel(gopts.ctx) + defer cancel() + for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) { + Verbosef("snapshot %s of %v at %s):\n", sn.ID().Str(), sn.Paths, sn.Time) - if snapshotIDString == "latest" { - id, err = restic.FindLatestSnapshot(repo, lsOptions.Paths, lsOptions.Host) - if err != nil { - Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, lsOptions.Paths, lsOptions.Host) - } - } else { - id, err = restic.FindSnapshot(repo, snapshotIDString) - if err != nil { - Exitf(1, "invalid id %q: %v", snapshotIDString, err) + if err = printTree(repo, sn.Tree, string(filepath.Separator)); err != nil { + return err } } - - sn, err := restic.LoadSnapshot(repo, id) - if err != nil { - return err - } - - Verbosef("snapshot of %v at %s:\n", sn.Paths, sn.Time) - - return printTree(string(filepath.Separator), repo, *sn.Tree) + return nil } diff --git a/src/cmds/restic/cmd_mount.go b/src/cmds/restic/cmd_mount.go index a4b8340c..20ce5dec 100644 --- a/src/cmds/restic/cmd_mount.go +++ b/src/cmds/restic/cmd_mount.go @@ -35,6 +35,9 @@ type MountOptions struct { OwnerRoot bool AllowRoot bool AllowOther bool + Host string + Tags []string + Paths []string } var mountOptions MountOptions @@ -42,9 +45,14 @@ var mountOptions MountOptions func init() { cmdRoot.AddCommand(cmdMount) - cmdMount.Flags().BoolVar(&mountOptions.OwnerRoot, "owner-root", false, "use 'root' as the owner of files and dirs") - cmdMount.Flags().BoolVar(&mountOptions.AllowRoot, "allow-root", false, "allow root user to access the data in the mounted directory") - cmdMount.Flags().BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory") + mountFlags := cmdMount.Flags() + mountFlags.BoolVar(&mountOptions.OwnerRoot, "owner-root", false, "use 'root' as the owner of files and dirs") + mountFlags.BoolVar(&mountOptions.AllowRoot, "allow-root", false, "allow root user to access the data in the mounted directory") + mountFlags.BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory") + + mountFlags.StringVarP(&mountOptions.Host, "host", "H", "", `only consider snapshots for this host`) + mountFlags.StringSliceVar(&mountOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`") + mountFlags.StringSliceVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`") } func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error { @@ -91,7 +99,7 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error { Printf("Don't forget to umount after quitting!\n") root := fs.Tree{} - root.Add("snapshots", fuse.NewSnapshotsDir(repo, opts.OwnerRoot)) + root.Add("snapshots", fuse.NewSnapshotsDir(repo, opts.OwnerRoot, opts.Paths, opts.Tags, opts.Host)) debug.Log("serving mount at %v", mountpoint) err = fs.Serve(c, &root) diff --git a/src/cmds/restic/cmd_prune.go b/src/cmds/restic/cmd_prune.go index fec8126f..03d14d30 100644 --- a/src/cmds/restic/cmd_prune.go +++ b/src/cmds/restic/cmd_prune.go @@ -1,8 +1,8 @@ package main import ( + "context" "fmt" - "os" "restic" "restic/debug" "restic/errors" @@ -81,8 +81,8 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error { return err } - done := make(chan struct{}) - defer close(done) + ctx, cancel := context.WithCancel(gopts.ctx) + defer cancel() var stats struct { blobs int @@ -92,7 +92,7 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error { } Verbosef("counting files in repo\n") - for _ = range repo.List(restic.DataFile, done) { + for _ = range repo.List(restic.DataFile, ctx.Done()) { stats.packs++ } @@ -238,35 +238,10 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error { bar.Done() } - Verbosef("creating new index\n") - - stats.packs = 0 - for _ = range repo.List(restic.DataFile, done) { - stats.packs++ - } - bar = newProgressMax(!gopts.Quiet, uint64(stats.packs), "packs") - idx, err = index.New(repo, bar) - if err != nil { + if err = rebuildIndex(ctx, repo); err != nil { return err } - var supersedes restic.IDs - for idxID := range repo.List(restic.IndexFile, done) { - h := restic.Handle{Type: restic.IndexFile, Name: idxID.String()} - err := repo.Backend().Remove(h) - if err != nil { - fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", idxID.Str(), err) - } - - supersedes = append(supersedes, idxID) - } - - id, err := idx.Save(repo, supersedes) - if err != nil { - return err - } - Verbosef("saved new index as %v\n", id.Str()) - Verbosef("done\n") return nil } diff --git a/src/cmds/restic/cmd_rebuild_index.go b/src/cmds/restic/cmd_rebuild_index.go index 6f6647da..e392b80c 100644 --- a/src/cmds/restic/cmd_rebuild_index.go +++ b/src/cmds/restic/cmd_rebuild_index.go @@ -1,6 +1,7 @@ package main import ( + "context" "restic" "restic/index" @@ -35,25 +36,29 @@ func runRebuildIndex(gopts GlobalOptions) error { return err } - done := make(chan struct{}) - defer close(done) + ctx, cancel := context.WithCancel(gopts.ctx) + defer cancel() + return rebuildIndex(ctx, repo) +} +func rebuildIndex(ctx context.Context, repo restic.Repository) error { Verbosef("counting files in repo\n") var packs uint64 - for _ = range repo.List(restic.DataFile, done) { + for _ = range repo.List(restic.DataFile, ctx.Done()) { packs++ } - bar := newProgressMax(!gopts.Quiet, packs, "packs") + bar := newProgressMax(!globalOptions.Quiet, packs, "packs") idx, err := index.New(repo, bar) if err != nil { return err } - Verbosef("listing old index files\n") + Verbosef("finding old index files\n") + var supersedes restic.IDs - for id := range repo.List(restic.IndexFile, done) { + for id := range repo.List(restic.IndexFile, ctx.Done()) { supersedes = append(supersedes, id) } @@ -67,13 +72,11 @@ func runRebuildIndex(gopts GlobalOptions) error { Verbosef("remove %d old index files\n", len(supersedes)) for _, id := range supersedes { - err := repo.Backend().Remove(restic.Handle{ + if err := repo.Backend().Remove(restic.Handle{ Type: restic.IndexFile, Name: id.String(), - }) - - if err != nil { - Warnf("error deleting old index %v: %v\n", id.Str(), err) + }); err != nil { + Warnf("error removing old index %v: %v\n", id.Str(), err) } } diff --git a/src/cmds/restic/cmd_restore.go b/src/cmds/restic/cmd_restore.go index 8a3e0757..6a9ec953 100644 --- a/src/cmds/restic/cmd_restore.go +++ b/src/cmds/restic/cmd_restore.go @@ -31,6 +31,7 @@ type RestoreOptions struct { Target string Host string Paths []string + Tags []string } var restoreOptions RestoreOptions @@ -44,6 +45,7 @@ func init() { flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to") flags.StringVarP(&restoreOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`) + flags.StringSliceVar(&restoreOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` for snapshot ID \"latest\"") flags.StringSliceVar(&restoreOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"") } @@ -85,7 +87,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error { var id restic.ID if snapshotIDString == "latest" { - id, err = restic.FindLatestSnapshot(repo, opts.Paths, opts.Host) + id, err = restic.FindLatestSnapshot(repo, opts.Paths, opts.Tags, opts.Host) if err != nil { Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, opts.Paths, opts.Host) } diff --git a/src/cmds/restic/cmd_snapshots.go b/src/cmds/restic/cmd_snapshots.go index 5c7a671b..7a3fa987 100644 --- a/src/cmds/restic/cmd_snapshots.go +++ b/src/cmds/restic/cmd_snapshots.go @@ -1,19 +1,19 @@ package main import ( + "context" + "encoding/json" "fmt" "io" - "restic/errors" "sort" "github.com/spf13/cobra" - "encoding/json" "restic" ) var cmdSnapshots = &cobra.Command{ - Use: "snapshots", + Use: "snapshots [snapshotID ...]", Short: "list all snapshots", Long: ` The "snapshots" command lists all snapshots stored in the repository. @@ -23,9 +23,10 @@ The "snapshots" command lists all snapshots stored in the repository. }, } -// SnapshotOptions bundle all options for the snapshots command. +// SnapshotOptions bundles all options for the snapshots command. type SnapshotOptions struct { Host string + Tags []string Paths []string } @@ -35,15 +36,12 @@ func init() { cmdRoot.AddCommand(cmdSnapshots) f := cmdSnapshots.Flags() - f.StringVar(&snapshotOptions.Host, "host", "", "only print snapshots for this host") - f.StringSliceVar(&snapshotOptions.Paths, "path", []string{}, "only print snapshots for this `path` (can be specified multiple times)") + f.StringVarP(&snapshotOptions.Host, "host", "H", "", "only consider snapshots for this `host`") + f.StringSliceVar(&snapshotOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` (can be specified multiple times)") + f.StringSliceVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)") } func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error { - if len(args) != 0 { - return errors.Fatal("wrong number of arguments") - } - repo, err := OpenRepository(gopts) if err != nil { return err @@ -57,32 +55,14 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro } } - done := make(chan struct{}) - defer close(done) - - list := []*restic.Snapshot{} - for id := range repo.List(restic.SnapshotFile, done) { - sn, err := restic.LoadSnapshot(repo, id) - if err != nil { - Warnf("error loading snapshot %s: %v\n", id, err) - continue - } - - if (opts.Host == "" || opts.Host == sn.Hostname) && sn.HasPaths(opts.Paths) { - pos := sort.Search(len(list), func(i int) bool { - return list[i].Time.After(sn.Time) - }) - - if pos < len(list) { - list = append(list, nil) - copy(list[pos+1:], list[pos:]) - list[pos] = sn - } else { - list = append(list, sn) - } - } + ctx, cancel := context.WithCancel(gopts.ctx) + defer cancel() + var list restic.Snapshots + for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) { + list = append(list, sn) } + sort.Sort(sort.Reverse(list)) if gopts.JSON { err := printSnapshotsJSON(gopts.stdout, list) @@ -97,7 +77,7 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro } // PrintSnapshots prints a text table of the snapshots in list to stdout. -func PrintSnapshots(stdout io.Writer, list []*restic.Snapshot) { +func PrintSnapshots(stdout io.Writer, list restic.Snapshots) { // Determine the max widths for host and tag. maxHost, maxTag := 10, 6 @@ -165,7 +145,7 @@ func PrintSnapshots(stdout io.Writer, list []*restic.Snapshot) { tab.Write(stdout) } -// Snapshot helps to print Snaphots as JSON +// Snapshot helps to print Snaphots as JSON with their ID included. type Snapshot struct { *restic.Snapshot @@ -173,7 +153,7 @@ type Snapshot struct { } // printSnapshotsJSON writes the JSON representation of list to stdout. -func printSnapshotsJSON(stdout io.Writer, list []*restic.Snapshot) error { +func printSnapshotsJSON(stdout io.Writer, list restic.Snapshots) error { var snapshots []Snapshot @@ -187,5 +167,4 @@ func printSnapshotsJSON(stdout io.Writer, list []*restic.Snapshot) error { } return json.NewEncoder(stdout).Encode(snapshots) - } diff --git a/src/cmds/restic/cmd_tag.go b/src/cmds/restic/cmd_tag.go index 1d8d5ebd..17ed8191 100644 --- a/src/cmds/restic/cmd_tag.go +++ b/src/cmds/restic/cmd_tag.go @@ -1,6 +1,8 @@ package main import ( + "context" + "github.com/spf13/cobra" "restic" @@ -45,22 +47,14 @@ func init() { tagFlags.StringSliceVar(&tagOptions.AddTags, "add", nil, "`tag` which will be added to the existing tags (can be given multiple times)") tagFlags.StringSliceVar(&tagOptions.RemoveTags, "remove", nil, "`tag` which will be removed from the existing tags (can be given multiple times)") - tagFlags.StringVarP(&tagOptions.Host, "host", "H", "", `only consider snapshots for this host, when no snapshot ID is given`) + tagFlags.StringVarP(&tagOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given") tagFlags.StringSliceVar(&tagOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given") tagFlags.StringSliceVar(&tagOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given") } -func changeTags(repo *repository.Repository, snapshotID restic.ID, setTags, addTags, removeTags, tags, paths []string, host string) (bool, error) { +func changeTags(repo *repository.Repository, sn *restic.Snapshot, setTags, addTags, removeTags []string) (bool, error) { var changed bool - sn, err := restic.LoadSnapshot(repo, snapshotID) - if err != nil { - return false, err - } - if (host != "" && host != sn.Hostname) || !sn.HasTags(tags) || !sn.HasPaths(paths) { - return false, nil - } - if len(setTags) != 0 { // Setting the tag to an empty string really means no tags. if len(setTags) == 1 && setTags[0] == "" { @@ -126,37 +120,13 @@ func runTag(opts TagOptions, gopts GlobalOptions, args []string) error { } } - var ids restic.IDs - if len(args) != 0 { - // When explit snapshot-IDs are given, the filtering does not matter anymore. - opts.Host = "" - opts.Tags = nil - opts.Paths = nil - - // Process all snapshot IDs given as arguments. - for _, s := range args { - snapshotID, err := restic.FindSnapshot(repo, s) - if err != nil { - Warnf("could not find a snapshot for ID %q, ignoring: %v\n", s, err) - continue - } - ids = append(ids, snapshotID) - } - ids = ids.Uniq() - } else { - // If there were no arguments, just get all snapshots. - done := make(chan struct{}) - defer close(done) - for snapshotID := range repo.List(restic.SnapshotFile, done) { - ids = append(ids, snapshotID) - } - } - changeCnt := 0 - for _, id := range ids { - changed, err := changeTags(repo, id, opts.SetTags, opts.AddTags, opts.RemoveTags, opts.Tags, opts.Paths, opts.Host) + ctx, cancel := context.WithCancel(gopts.ctx) + defer cancel() + for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) { + changed, err := changeTags(repo, sn, opts.SetTags, opts.AddTags, opts.RemoveTags) if err != nil { - Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", id, err) + Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err) continue } if changed { diff --git a/src/cmds/restic/find.go b/src/cmds/restic/find.go new file mode 100644 index 00000000..fa9d7169 --- /dev/null +++ b/src/cmds/restic/find.go @@ -0,0 +1,78 @@ +package main + +import ( + "context" + + "restic" + "restic/repository" +) + +// FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. +func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, host string, tags []string, paths []string, snapshotIDs []string) <-chan *restic.Snapshot { + out := make(chan *restic.Snapshot) + go func() { + defer close(out) + if len(snapshotIDs) != 0 { + var ( + id restic.ID + usedFilter bool + err error + ) + ids := make(restic.IDs, 0, len(snapshotIDs)) + // Process all snapshot IDs given as arguments. + for _, s := range snapshotIDs { + if s == "latest" { + id, err = restic.FindLatestSnapshot(repo, paths, tags, host) + if err != nil { + Warnf("Ignoring %q, no snapshot matched given filter (Paths:%v Tags:%v Host:%v)\n", s, paths, tags, host) + usedFilter = true + continue + } + } else { + id, err = restic.FindSnapshot(repo, s) + if err != nil { + Warnf("Ignoring %q, it is not a snapshot id\n", s) + continue + } + } + ids = append(ids, id) + } + + // Give the user some indication their filters are not used. + if !usedFilter && (host != "" || len(tags) != 0 || len(paths) != 0) { + Warnf("Ignoring filters as there are explicit snapshot ids given\n") + } + + for _, id := range ids.Uniq() { + sn, err := restic.LoadSnapshot(repo, id) + if err != nil { + Warnf("Ignoring %q, could not load snapshot: %v\n", id, err) + continue + } + select { + case <-ctx.Done(): + return + case out <- sn: + } + } + return + } + + for id := range repo.List(restic.SnapshotFile, ctx.Done()) { + sn, err := restic.LoadSnapshot(repo, id) + if err != nil { + Warnf("Ignoring %q, could not load snapshot: %v\n", id, err) + continue + } + if (host != "" && host != sn.Hostname) || !sn.HasTags(tags) || !sn.HasPaths(paths) { + continue + } + select { + case <-ctx.Done(): + return + case out <- sn: + } + } + }() + return out +} diff --git a/src/cmds/restic/format.go b/src/cmds/restic/format.go index 68fa29fb..16c37469 100644 --- a/src/cmds/restic/format.go +++ b/src/cmds/restic/format.go @@ -2,7 +2,11 @@ package main import ( "fmt" + "os" + "path/filepath" "time" + + "restic" ) func formatBytes(c uint64) string { @@ -58,3 +62,23 @@ func formatDuration(d time.Duration) string { sec := uint64(d / time.Second) return formatSeconds(sec) } + +func formatNode(prefix string, n *restic.Node, long bool) string { + if !long { + return filepath.Join(prefix, n.Name) + } + + switch n.Type { + case "file": + return fmt.Sprintf("%s %5d %5d %6d %s %s", + n.Mode, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name)) + case "dir": + return fmt.Sprintf("%s %5d %5d %6d %s %s", + n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name)) + case "symlink": + return fmt.Sprintf("%s %5d %5d %6d %s %s -> %s", + n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name), n.LinkTarget) + default: + return fmt.Sprintf("", n.Type, n.Name) + } +} diff --git a/src/cmds/restic/global.go b/src/cmds/restic/global.go index 02a2e9b5..4acd7906 100644 --- a/src/cmds/restic/global.go +++ b/src/cmds/restic/global.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "io" "io/ioutil" @@ -33,6 +34,7 @@ type GlobalOptions struct { NoLock bool JSON bool + ctx context.Context password string stdout io.Writer stderr io.Writer @@ -49,6 +51,13 @@ func init() { globalOptions.password = pw } + var cancel context.CancelFunc + globalOptions.ctx, cancel = context.WithCancel(context.Background()) + AddCleanupHandler(func() error { + cancel() + return nil + }) + f := cmdRoot.PersistentFlags() f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "repository to backup to or restore from (default: $RESTIC_REPOSITORY)") f.StringVarP(&globalOptions.PasswordFile, "password-file", "p", "", "read the repository password from a file") diff --git a/src/cmds/restic/integration_helpers_test.go b/src/cmds/restic/integration_helpers_test.go index c71b1206..ad6acc8a 100644 --- a/src/cmds/restic/integration_helpers_test.go +++ b/src/cmds/restic/integration_helpers_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "io/ioutil" "os" @@ -194,6 +195,7 @@ func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions)) gopts := GlobalOptions{ Repo: env.repo, Quiet: true, + ctx: context.Background(), password: TestPassword, stdout: os.Stdout, stderr: os.Stderr, diff --git a/src/cmds/restic/integration_test.go b/src/cmds/restic/integration_test.go index 9d7ea55f..0adf495a 100644 --- a/src/cmds/restic/integration_test.go +++ b/src/cmds/restic/integration_test.go @@ -142,7 +142,9 @@ func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string { globalOptions.Quiet = quiet }() - OK(t, runLs(gopts, []string{snapshotID})) + opts := LsOptions{} + + OK(t, runLs(opts, gopts, []string{snapshotID})) return strings.Split(string(buf.Bytes()), "\n") } diff --git a/src/restic/fuse/snapshot.go b/src/restic/fuse/snapshot.go index 1e1092de..2a654397 100644 --- a/src/restic/fuse/snapshot.go +++ b/src/restic/fuse/snapshot.go @@ -32,6 +32,9 @@ var _ = fs.NodeStringLookuper(&SnapshotsDir{}) type SnapshotsDir struct { repo restic.Repository ownerIsRoot bool + paths []string + tags []string + host string // knownSnapshots maps snapshot timestamp to the snapshot sync.RWMutex @@ -40,12 +43,15 @@ type SnapshotsDir struct { } // NewSnapshotsDir returns a new dir object for the snapshots. -func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool) *SnapshotsDir { +func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool, paths []string, tags []string, host string) *SnapshotsDir { debug.Log("fuse mount initiated") return &SnapshotsDir{ repo: repo, - knownSnapshots: make(map[string]SnapshotWithId), ownerIsRoot: ownerIsRoot, + paths: paths, + tags: tags, + host: host, + knownSnapshots: make(map[string]SnapshotWithId), processed: restic.NewIDSet(), } } @@ -79,6 +85,13 @@ func (sn *SnapshotsDir) updateCache(ctx context.Context) error { return err } + // Filter snapshots we don't care for. + if (sn.host != "" && sn.host != snapshot.Hostname) || + !snapshot.HasTags(sn.tags) || + !snapshot.HasPaths(sn.paths) { + continue + } + timestamp := snapshot.Time.Format(time.RFC3339) for i := 1; ; i++ { if _, ok := sn.knownSnapshots[timestamp]; !ok { diff --git a/src/restic/snapshot.go b/src/restic/snapshot.go index ed89a60c..343dfe41 100644 --- a/src/restic/snapshot.go +++ b/src/restic/snapshot.go @@ -177,8 +177,8 @@ func (sn *Snapshot) SamePaths(paths []string) bool { // ErrNoSnapshotFound is returned when no snapshot for the given criteria could be found. var ErrNoSnapshotFound = errors.New("no snapshot found") -// FindLatestSnapshot finds latest snapshot with optional target/directory and hostname filters. -func FindLatestSnapshot(repo Repository, targets []string, hostname string) (ID, error) { +// FindLatestSnapshot finds latest snapshot with optional target/directory, tags and hostname filters. +func FindLatestSnapshot(repo Repository, targets []string, tags []string, hostname string) (ID, error) { var ( latest time.Time latestID ID @@ -190,7 +190,7 @@ func FindLatestSnapshot(repo Repository, targets []string, hostname string) (ID, if err != nil { return ID{}, errors.Errorf("Error listing snapshot: %v", err) } - if snapshot.Time.After(latest) && snapshot.HasPaths(targets) && (hostname == "" || hostname == snapshot.Hostname) { + if snapshot.Time.After(latest) && (hostname == "" || hostname == snapshot.Hostname) && snapshot.HasTags(tags) && snapshot.HasPaths(targets) { latest = snapshot.Time latestID = snapshotID found = true