From 31ff506309ce8063a6eef4e03b287cd33f707098 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Mon, 27 Feb 2017 19:42:00 +0100
Subject: [PATCH 01/28] Ignore empty lines in --files-from

Closes #822
---
 src/cmds/restic/cmd_backup.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/cmds/restic/cmd_backup.go b/src/cmds/restic/cmd_backup.go
index dca861298..3d2514ed3 100644
--- a/src/cmds/restic/cmd_backup.go
+++ b/src/cmds/restic/cmd_backup.go
@@ -304,7 +304,11 @@ func readLinesFromFile(filename string) ([]string, error) {
 
 	scanner := bufio.NewScanner(r)
 	for scanner.Scan() {
-		lines = append(lines, scanner.Text())
+		line := scanner.Text()
+		if line == "" {
+			continue
+		}
+		lines = append(lines, line)
 	}
 
 	if err := scanner.Err(); err != nil {

From 80e93621e1b6b9e2406b2bb55d2e76c6d1130e7d Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Mon, 27 Feb 2017 20:37:47 +0100
Subject: [PATCH 02/28] Travis: Configure Go versions

---
 .travis.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 63d7c5a32..d25b0be4c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -19,13 +19,15 @@ matrix:
   exclude:
     - os: osx
       go: 1.6.4
+    - os: osx
+      go: 1.7.5
     - os: osx
       go: tip
     - os: linux
       go: 1.7.5
   include:
     - os: linux
-      go: 1.7.5
+      go: 1.8
       sudo: true
       env:
         RESTIC_TEST_FUSE=1

From bb69b20affc479d6169a4f9589c9cca2b7f09d7a Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Mon, 27 Feb 2017 20:39:00 +0100
Subject: [PATCH 03/28] Travis: Configure Go versions

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index d25b0be4c..e6313bd00 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,7 +24,7 @@ matrix:
     - os: osx
       go: tip
     - os: linux
-      go: 1.7.5
+      go: 1.6.4
   include:
     - os: linux
       go: 1.8

From 92ad35848a586a690831d1f6118c925083fa00c9 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Mon, 27 Feb 2017 20:40:10 +0100
Subject: [PATCH 04/28] Fix Travis Go versions

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index e6313bd00..f2d6b6bb9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,7 +24,7 @@ matrix:
     - os: osx
       go: tip
     - os: linux
-      go: 1.6.4
+      go: 1.8
   include:
     - os: linux
       go: 1.8

From bf97cc7efa36b45fa9885b2f18b3434adff3fe45 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Thu, 2 Mar 2017 14:50:54 +0100
Subject: [PATCH 05/28] Allow filtering absolute paths

Before, the restorer called the filter function with a relative path,
this prevented anchoring absolute patterns (which just never matched).
Now call the restore function with an absolute virtual path, starting at
the filepath separator.

Closes #834
---
 src/restic/restorer.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/restic/restorer.go b/src/restic/restorer.go
index 4a271cec0..2d1513bb7 100644
--- a/src/restic/restorer.go
+++ b/src/restic/restorer.go
@@ -120,7 +120,7 @@ func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string, idx *Hard
 // Before an item is created, res.Filter is called.
 func (res *Restorer) RestoreTo(dir string) error {
 	idx := NewHardlinkIndex()
-	return res.restoreTo(dir, "", *res.sn.Tree, idx)
+	return res.restoreTo(dir, string(filepath.Separator), *res.sn.Tree, idx)
 }
 
 // Snapshot returns the snapshot this restorer is configured to use.

From 7c92994f1062b6f19df0f2dfa1a54c04b14cc2d9 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Thu, 2 Mar 2017 14:52:18 +0100
Subject: [PATCH 06/28] Clarify variable name

---
 src/restic/restorer.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/restic/restorer.go b/src/restic/restorer.go
index 2d1513bb7..56916f3ce 100644
--- a/src/restic/restorer.go
+++ b/src/restic/restorer.go
@@ -116,11 +116,11 @@ func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string, idx *Hard
 	return nil
 }
 
-// RestoreTo creates the directories and files in the snapshot below dir.
+// RestoreTo creates the directories and files in the snapshot below dst.
 // Before an item is created, res.Filter is called.
-func (res *Restorer) RestoreTo(dir string) error {
+func (res *Restorer) RestoreTo(dst string) error {
 	idx := NewHardlinkIndex()
-	return res.restoreTo(dir, string(filepath.Separator), *res.sn.Tree, idx)
+	return res.restoreTo(dst, string(filepath.Separator), *res.sn.Tree, idx)
 }
 
 // Snapshot returns the snapshot this restorer is configured to use.

From 9b776dc7abc4a2262a1577ed76c9a8355d857517 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Wed, 1 Mar 2017 13:44:56 +0100
Subject: [PATCH 07/28] Use new Index implementation for rebuild-index

---
 src/cmds/restic/cmd_rebuild_index.go   | 47 +++++++++++++++++-
 src/restic/repository/index_rebuild.go | 66 --------------------------
 src/restic/repository/repack_test.go   | 21 +++++++-
 3 files changed, 64 insertions(+), 70 deletions(-)
 delete mode 100644 src/restic/repository/index_rebuild.go

diff --git a/src/cmds/restic/cmd_rebuild_index.go b/src/cmds/restic/cmd_rebuild_index.go
index e4a6c680a..6f6647daa 100644
--- a/src/cmds/restic/cmd_rebuild_index.go
+++ b/src/cmds/restic/cmd_rebuild_index.go
@@ -1,7 +1,8 @@
 package main
 
 import (
-	"restic/repository"
+	"restic"
+	"restic/index"
 
 	"github.com/spf13/cobra"
 )
@@ -34,5 +35,47 @@ func runRebuildIndex(gopts GlobalOptions) error {
 		return err
 	}
 
-	return repository.RebuildIndex(repo)
+	done := make(chan struct{})
+	defer close(done)
+
+	Verbosef("counting files in repo\n")
+
+	var packs uint64
+	for _ = range repo.List(restic.DataFile, done) {
+		packs++
+	}
+
+	bar := newProgressMax(!gopts.Quiet, packs, "packs")
+	idx, err := index.New(repo, bar)
+	if err != nil {
+		return err
+	}
+
+	Verbosef("listing old index files\n")
+	var supersedes restic.IDs
+	for id := range repo.List(restic.IndexFile, done) {
+		supersedes = append(supersedes, id)
+	}
+
+	id, err := idx.Save(repo, supersedes)
+	if err != nil {
+		return err
+	}
+
+	Verbosef("saved new index as %v\n", id.Str())
+
+	Verbosef("remove %d old index files\n", len(supersedes))
+
+	for _, id := range supersedes {
+		err := repo.Backend().Remove(restic.Handle{
+			Type: restic.IndexFile,
+			Name: id.String(),
+		})
+
+		if err != nil {
+			Warnf("error deleting old index %v: %v\n", id.Str(), err)
+		}
+	}
+
+	return nil
 }
diff --git a/src/restic/repository/index_rebuild.go b/src/restic/repository/index_rebuild.go
deleted file mode 100644
index fcfc3027f..000000000
--- a/src/restic/repository/index_rebuild.go
+++ /dev/null
@@ -1,66 +0,0 @@
-package repository
-
-import (
-	"fmt"
-	"os"
-	"restic"
-	"restic/debug"
-	"restic/list"
-	"restic/worker"
-)
-
-// RebuildIndex lists all packs in the repo, writes a new index and removes all
-// old indexes. This operation should only be done with an exclusive lock in
-// place.
-func RebuildIndex(repo restic.Repository) error {
-	debug.Log("start rebuilding index")
-
-	done := make(chan struct{})
-	defer close(done)
-
-	ch := make(chan worker.Job)
-	go list.AllPacks(repo, ch, done)
-
-	idx := NewIndex()
-	for job := range ch {
-		id := job.Data.(restic.ID)
-
-		if job.Error != nil {
-			fmt.Fprintf(os.Stderr, "error for pack %v: %v\n", id, job.Error)
-			continue
-		}
-
-		res := job.Result.(list.Result)
-
-		for _, entry := range res.Entries() {
-			pb := restic.PackedBlob{
-				Blob:   entry,
-				PackID: res.PackID(),
-			}
-			idx.Store(pb)
-		}
-	}
-
-	oldIndexes := restic.NewIDSet()
-	for id := range repo.List(restic.IndexFile, done) {
-		idx.AddToSupersedes(id)
-		oldIndexes.Insert(id)
-	}
-
-	id, err := SaveIndex(repo, idx)
-	if err != nil {
-		debug.Log("error saving index: %v", err)
-		return err
-	}
-	debug.Log("new index saved as %v", id.Str())
-
-	for indexID := range oldIndexes {
-		h := restic.Handle{Type: restic.IndexFile, Name: indexID.String()}
-		err := repo.Backend().Remove(h)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", indexID.Str(), err)
-		}
-	}
-
-	return nil
-}
diff --git a/src/restic/repository/repack_test.go b/src/restic/repository/repack_test.go
index 6d910c97b..619454336 100644
--- a/src/restic/repository/repack_test.go
+++ b/src/restic/repository/repack_test.go
@@ -4,6 +4,7 @@ import (
 	"io"
 	"math/rand"
 	"restic"
+	"restic/index"
 	"restic/repository"
 	"testing"
 )
@@ -144,8 +145,24 @@ func saveIndex(t *testing.T, repo restic.Repository) {
 }
 
 func rebuildIndex(t *testing.T, repo restic.Repository) {
-	if err := repository.RebuildIndex(repo); err != nil {
-		t.Fatalf("error rebuilding index: %v", err)
+	idx, err := index.New(repo, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for id := range repo.List(restic.IndexFile, nil) {
+		err = repo.Backend().Remove(restic.Handle{
+			Type: restic.IndexFile,
+			Name: id.String(),
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	_, err = idx.Save(repo, nil)
+	if err != nil {
+		t.Fatal(err)
 	}
 }
 

From f53d33ba34ae93383f037605d583d18cc3c58ff6 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Thu, 2 Mar 2017 15:45:35 +0100
Subject: [PATCH 08/28] Make ArchiveReader a struct

---
 src/cmds/restic/cmd_backup.go              |  8 +++++-
 src/restic/archiver/archive_reader.go      | 16 ++++++++---
 src/restic/archiver/archive_reader_test.go | 32 +++++++++++++++++++---
 3 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/src/cmds/restic/cmd_backup.go b/src/cmds/restic/cmd_backup.go
index 3d2514ed3..6c259ffa0 100644
--- a/src/cmds/restic/cmd_backup.go
+++ b/src/cmds/restic/cmd_backup.go
@@ -272,7 +272,13 @@ func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string)
 		return err
 	}
 
-	_, id, err := archiver.ArchiveReader(repo, newArchiveStdinProgress(gopts), os.Stdin, opts.StdinFilename, opts.Tags, opts.Hostname)
+	r := &archiver.Reader{
+		Repository: repo,
+		Tags:       opts.Tags,
+		Hostname:   opts.Hostname,
+	}
+
+	_, id, err := r.Archive(opts.StdinFilename, os.Stdin, newArchiveStdinProgress(gopts))
 	if err != nil {
 		return err
 	}
diff --git a/src/restic/archiver/archive_reader.go b/src/restic/archiver/archive_reader.go
index 43beee69a..6ed72ab96 100644
--- a/src/restic/archiver/archive_reader.go
+++ b/src/restic/archiver/archive_reader.go
@@ -11,15 +11,22 @@ import (
 	"github.com/restic/chunker"
 )
 
-// ArchiveReader reads from the reader and archives the data. Returned is the
-// resulting snapshot and its ID.
-func ArchiveReader(repo restic.Repository, p *restic.Progress, rd io.Reader, name string, tags []string, hostname string) (*restic.Snapshot, restic.ID, error) {
+// Reader allows saving a stream of data to the repository.
+type Reader struct {
+	restic.Repository
+
+	Tags     []string
+	Hostname string
+}
+
+// Archive reads data from the reader and saves it to the repo.
+func (r *Reader) Archive(name string, rd io.Reader, p *restic.Progress) (*restic.Snapshot, restic.ID, error) {
 	if name == "" {
 		return nil, restic.ID{}, errors.New("no filename given")
 	}
 
 	debug.Log("start archiving %s", name)
-	sn, err := restic.NewSnapshot([]string{name}, tags, hostname)
+	sn, err := restic.NewSnapshot([]string{name}, r.Tags, r.Hostname)
 	if err != nil {
 		return nil, restic.ID{}, err
 	}
@@ -27,6 +34,7 @@ func ArchiveReader(repo restic.Repository, p *restic.Progress, rd io.Reader, nam
 	p.Start()
 	defer p.Done()
 
+	repo := r.Repository
 	chnker := chunker.New(rd, repo.Config().ChunkerPolynomial)
 
 	ids := restic.IDs{}
diff --git a/src/restic/archiver/archive_reader_test.go b/src/restic/archiver/archive_reader_test.go
index bdcc2a1e8..a8ab18668 100644
--- a/src/restic/archiver/archive_reader_test.go
+++ b/src/restic/archiver/archive_reader_test.go
@@ -79,7 +79,13 @@ func TestArchiveReader(t *testing.T) {
 
 	f := fakeFile(t, seed, size)
 
-	sn, id, err := ArchiveReader(repo, nil, f, "fakefile", []string{"test"}, "localhost")
+	r := &Reader{
+		Repository: repo,
+		Hostname:   "localhost",
+		Tags:       []string{"test"},
+	}
+
+	sn, id, err := r.Archive("fakefile", f, nil)
 	if err != nil {
 		t.Fatalf("ArchiveReader() returned error %v", err)
 	}
@@ -99,7 +105,13 @@ func TestArchiveReaderNull(t *testing.T) {
 	repo, cleanup := repository.TestRepository(t)
 	defer cleanup()
 
-	sn, id, err := ArchiveReader(repo, nil, bytes.NewReader(nil), "fakefile", nil, "localhost")
+	r := &Reader{
+		Repository: repo,
+		Hostname:   "localhost",
+		Tags:       []string{"test"},
+	}
+
+	sn, id, err := r.Archive("fakefile", bytes.NewReader(nil), nil)
 	if err != nil {
 		t.Fatalf("ArchiveReader() returned error %v", err)
 	}
@@ -134,7 +146,13 @@ func TestArchiveReaderError(t *testing.T) {
 	repo, cleanup := repository.TestRepository(t)
 	defer cleanup()
 
-	sn, id, err := ArchiveReader(repo, nil, errReader("error returned by reading stdin"), "fakefile", nil, "localhost")
+	r := &Reader{
+		Repository: repo,
+		Hostname:   "localhost",
+		Tags:       []string{"test"},
+	}
+
+	sn, id, err := r.Archive("fakefile", errReader("error returned by reading stdin"), nil)
 	if err == nil {
 		t.Errorf("expected error not returned")
 	}
@@ -167,11 +185,17 @@ func BenchmarkArchiveReader(t *testing.B) {
 		t.Fatal(err)
 	}
 
+	r := &Reader{
+		Repository: repo,
+		Hostname:   "localhost",
+		Tags:       []string{"test"},
+	}
+
 	t.SetBytes(size)
 	t.ResetTimer()
 
 	for i := 0; i < t.N; i++ {
-		_, _, err := ArchiveReader(repo, nil, bytes.NewReader(buf), "fakefile", []string{"test"}, "localhost")
+		_, _, err := r.Archive("fakefile", bytes.NewReader(buf), nil)
 		if err != nil {
 			t.Fatal(err)
 		}

From efb4315a1e584d87c6ceed05daf7aeae581aca5a Mon Sep 17 00:00:00 2001
From: Pauline Middelink <middelink@polyware.nl>
Date: Thu, 2 Mar 2017 22:41:11 +0100
Subject: [PATCH 09/28] Display absolute paths when displaying the output of ls
 and find.

---
 src/cmds/restic/cmd_find.go | 2 +-
 src/cmds/restic/cmd_ls.go   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/cmds/restic/cmd_find.go b/src/cmds/restic/cmd_find.go
index 0eef529a6..ab628c414 100644
--- a/src/cmds/restic/cmd_find.go
+++ b/src/cmds/restic/cmd_find.go
@@ -129,7 +129,7 @@ func findInSnapshot(repo *repository.Repository, pat findPattern, id restic.ID)
 		return err
 	}
 
-	results, err := findInTree(repo, pat, *sn.Tree, "")
+	results, err := findInTree(repo, pat, *sn.Tree, string(filepath.Separator))
 	if err != nil {
 		return err
 	}
diff --git a/src/cmds/restic/cmd_ls.go b/src/cmds/restic/cmd_ls.go
index 2dfa5cc52..c6c05bec1 100644
--- a/src/cmds/restic/cmd_ls.go
+++ b/src/cmds/restic/cmd_ls.go
@@ -121,5 +121,5 @@ func runLs(gopts GlobalOptions, args []string) error {
 
 	Verbosef("snapshot of %v at %s:\n", sn.Paths, sn.Time)
 
-	return printTree("", repo, *sn.Tree)
+	return printTree(string(filepath.Separator), repo, *sn.Tree)
 }

From 039e81b04b8812463c03257bb293fb1eb44f6daa Mon Sep 17 00:00:00 2001
From: Pauline Middelink <middelink@polyware.nl>
Date: Thu, 2 Mar 2017 23:30:56 +0100
Subject: [PATCH 10/28] Fix unit test, we need to check for absolute paths now.

---
 src/cmds/restic/integration_test.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/cmds/restic/integration_test.go b/src/cmds/restic/integration_test.go
index 477e8052a..0a819e7bb 100644
--- a/src/cmds/restic/integration_test.go
+++ b/src/cmds/restic/integration_test.go
@@ -516,23 +516,23 @@ func TestBackupExclude(t *testing.T) {
 		testRunBackup(t, []string{datadir}, opts, gopts)
 		snapshots, snapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, gopts))
 		files := testRunLs(t, gopts, snapshotID)
-		Assert(t, includes(files, filepath.Join("testdata", "foo.tar.gz")),
+		Assert(t, includes(files, filepath.Join("/testdata", "foo.tar.gz")),
 			"expected file %q in first snapshot, but it's not included", "foo.tar.gz")
 
 		opts.Excludes = []string{"*.tar.gz"}
 		testRunBackup(t, []string{datadir}, opts, gopts)
 		snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, gopts))
 		files = testRunLs(t, gopts, snapshotID)
-		Assert(t, !includes(files, filepath.Join("testdata", "foo.tar.gz")),
+		Assert(t, !includes(files, filepath.Join("/testdata", "foo.tar.gz")),
 			"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
 
 		opts.Excludes = []string{"*.tar.gz", "private/secret"}
 		testRunBackup(t, []string{datadir}, opts, gopts)
 		snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, gopts))
 		files = testRunLs(t, gopts, snapshotID)
-		Assert(t, !includes(files, filepath.Join("testdata", "foo.tar.gz")),
+		Assert(t, !includes(files, filepath.Join("/testdata", "foo.tar.gz")),
 			"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
-		Assert(t, !includes(files, filepath.Join("testdata", "private", "secret", "passwords.txt")),
+		Assert(t, !includes(files, filepath.Join("/testdata", "private", "secret", "passwords.txt")),
 			"expected file %q not in first snapshot, but it's included", "passwords.txt")
 	})
 }

From 5ecaaea90befb70613a9a37ff124cff62033f48b Mon Sep 17 00:00:00 2001
From: Pauline Middelink <middelink@polyware.nl>
Date: Fri, 3 Mar 2017 11:14:39 +0100
Subject: [PATCH 11/28] Really use absolute pathnames, not all systems use /.

---
 src/cmds/restic/integration_test.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/cmds/restic/integration_test.go b/src/cmds/restic/integration_test.go
index 0a819e7bb..3c7533ac5 100644
--- a/src/cmds/restic/integration_test.go
+++ b/src/cmds/restic/integration_test.go
@@ -516,23 +516,23 @@ func TestBackupExclude(t *testing.T) {
 		testRunBackup(t, []string{datadir}, opts, gopts)
 		snapshots, snapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, gopts))
 		files := testRunLs(t, gopts, snapshotID)
-		Assert(t, includes(files, filepath.Join("/testdata", "foo.tar.gz")),
+		Assert(t, includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
 			"expected file %q in first snapshot, but it's not included", "foo.tar.gz")
 
 		opts.Excludes = []string{"*.tar.gz"}
 		testRunBackup(t, []string{datadir}, opts, gopts)
 		snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, gopts))
 		files = testRunLs(t, gopts, snapshotID)
-		Assert(t, !includes(files, filepath.Join("/testdata", "foo.tar.gz")),
+		Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
 			"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
 
 		opts.Excludes = []string{"*.tar.gz", "private/secret"}
 		testRunBackup(t, []string{datadir}, opts, gopts)
 		snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, gopts))
 		files = testRunLs(t, gopts, snapshotID)
-		Assert(t, !includes(files, filepath.Join("/testdata", "foo.tar.gz")),
+		Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
 			"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
-		Assert(t, !includes(files, filepath.Join("/testdata", "private", "secret", "passwords.txt")),
+		Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "private", "secret", "passwords.txt")),
 			"expected file %q not in first snapshot, but it's included", "passwords.txt")
 	})
 }

From 12ed2f65e36e094175be9a2b0527e02faed19689 Mon Sep 17 00:00:00 2001
From: welpo <welpo@users.noreply.github.com>
Date: Sat, 4 Mar 2017 13:19:50 +0100
Subject: [PATCH 12/28] Fix Minio Server URL

The previous link gave a 404
---
 doc/Manual.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/Manual.md b/doc/Manual.md
index 3933203e5..31133a309 100644
--- a/doc/Manual.md
+++ b/doc/Manual.md
@@ -526,7 +526,7 @@ only available via HTTP, you can specify the URL to the server like this:
 
 ### Pre-Requisites
 
-* Download and Install [Minio Server](https://minio.io/download/).
+* Download and Install [Minio Server](https://minio.io/downloads/#minio-server).
 * You can also refer to [https://docs.minio.io](https://docs.minio.io) for step by step guidance on installation and getting started on Minio Client and Minio Server.
 
 You must first setup the following environment variables with the credentials of your running Minio Server.

From 82458d4de0f6e9491bb75c866740ec1092dc1bc7 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sat, 4 Mar 2017 14:23:15 +0100
Subject: [PATCH 13/28] Update github.com/pkg/xattr

Closes #843
---
 vendor/manifest                               |  2 +-
 vendor/src/github.com/pkg/xattr/README.md     | 12 ++++--
 vendor/src/github.com/pkg/xattr/xattr.go      | 10 ++++-
 .../src/github.com/pkg/xattr/xattr_darwin.go  | 43 ++++++++++---------
 .../src/github.com/pkg/xattr/xattr_freebsd.go | 42 ++++++++++--------
 .../src/github.com/pkg/xattr/xattr_linux.go   | 41 ++++++++++--------
 6 files changed, 88 insertions(+), 62 deletions(-)

diff --git a/vendor/manifest b/vendor/manifest
index 53d94c88d..ec75a1f99 100644
--- a/vendor/manifest
+++ b/vendor/manifest
@@ -52,7 +52,7 @@
 		{
 			"importpath": "github.com/pkg/xattr",
 			"repository": "https://github.com/pkg/xattr",
-			"revision": "1d40b70a947cd8e8457e4715e1123f8e99f5f241",
+			"revision": "b867675798fa7708a444945602b452ca493f2272",
 			"branch": "master"
 		},
 		{
diff --git a/vendor/src/github.com/pkg/xattr/README.md b/vendor/src/github.com/pkg/xattr/README.md
index 9fb2e0d00..77d89567f 100644
--- a/vendor/src/github.com/pkg/xattr/README.md
+++ b/vendor/src/github.com/pkg/xattr/README.md
@@ -1,3 +1,7 @@
+[![GoDoc](https://godoc.org/github.com/pkg/xattr?status.svg)](http://godoc.org/github.com/pkg/xattr)
+[![Go Report Card](https://goreportcard.com/badge/github.com/pkg/xattr)](https://goreportcard.com/report/github.com/pkg/xattr)
+[![Build Status](https://travis-ci.org/pkg/xattr.svg?branch=master)](https://travis-ci.org/pkg/xattr)
+
 xattr
 =====
 Extended attribute support for Go (linux + darwin + freebsd).
@@ -10,12 +14,12 @@ Extended attribute support for Go (linux + darwin + freebsd).
   const path = "/tmp/myfile"
   const prefix = "user."
 
-  if err = Setxattr(path, prefix+"test", []byte("test-attr-value")); err != nil {
-    t.Fatal(err)
+  if err := xattr.Setxattr(path, prefix+"test", []byte("test-attr-value")); err != nil {
+    log.Fatal(err)
   }
 
   var data []byte
-  data, err = Getxattr(path, prefix+"test"); err != nil {
-    t.Fatal(err)
+  data, err = xattr.Getxattr(path, prefix+"test"); err != nil {
+    log.Fatal(err)
   }
 ```
diff --git a/vendor/src/github.com/pkg/xattr/xattr.go b/vendor/src/github.com/pkg/xattr/xattr.go
index a13007f8b..a4f06b7c3 100644
--- a/vendor/src/github.com/pkg/xattr/xattr.go
+++ b/vendor/src/github.com/pkg/xattr/xattr.go
@@ -1,3 +1,10 @@
+/*
+Package xattr provides support for extended attributes on linux, darwin and freebsd.
+Extended attributes are name:value pairs associated permanently with files and directories, 
+similar to the environment strings associated with a process. 
+An attribute may be defined or undefined. If it is defined, its value may be empty or non-empty.
+More details you can find here: https://en.wikipedia.org/wiki/Extended_file_attributes
+*/
 package xattr
 
 // XAttrError records an error and the operation, file path and attribute that caused it.
@@ -12,8 +19,7 @@ func (e *XAttrError) Error() string {
 	return e.Op + " " + e.Path + " " + e.Name + ": " + e.Err.Error()
 }
 
-// Convert an array of NULL terminated UTF-8 strings
-// to a []string.
+// nullTermToStrings converts an array of NULL terminated UTF-8 strings to a []string.
 func nullTermToStrings(buf []byte) (result []string) {
 	offset := 0
 	for index, b := range buf {
diff --git a/vendor/src/github.com/pkg/xattr/xattr_darwin.go b/vendor/src/github.com/pkg/xattr/xattr_darwin.go
index 443d049f2..07c1228e3 100644
--- a/vendor/src/github.com/pkg/xattr/xattr_darwin.go
+++ b/vendor/src/github.com/pkg/xattr/xattr_darwin.go
@@ -2,44 +2,47 @@
 
 package xattr
 
-// Retrieve extended attribute data associated with path.
+// Getxattr retrieves extended attribute data associated with path.
 func Getxattr(path, name string) ([]byte, error) {
 	// find size.
 	size, err := getxattr(path, name, nil, 0, 0, 0)
 	if err != nil {
 		return nil, &XAttrError{"getxattr", path, name, err}
 	}
-	buf := make([]byte, size)
-	// Read into buffer of that size.
-	read, err := getxattr(path, name, &buf[0], size, 0, 0)
-	if err != nil {
-		return nil, &XAttrError{"getxattr", path, name, err}
+	if size > 0 {
+		buf := make([]byte, size)
+		// Read into buffer of that size.
+		read, err := getxattr(path, name, &buf[0], size, 0, 0)
+		if err != nil {
+			return nil, &XAttrError{"getxattr", path, name, err}
+		}
+		return buf[:read], nil
 	}
-	return buf[:read], nil
+	return []byte{}, nil
 }
 
-// Retrieves a list of names of extended attributes associated with the
-// given path in the file system.
+// Listxattr retrieves a list of names of extended attributes associated
+// with the given path in the file system.
 func Listxattr(path string) ([]string, error) {
 	// find size.
 	size, err := listxattr(path, nil, 0, 0)
 	if err != nil {
 		return nil, &XAttrError{"listxattr", path, "", err}
 	}
-	if size == 0 {
-		return []string{}, nil
-	}
+	if size > 0 {
 
-	buf := make([]byte, size)
-	// Read into buffer of that size.
-	read, err := listxattr(path, &buf[0], size, 0)
-	if err != nil {
-		return nil, &XAttrError{"listxattr", path, "", err}
+		buf := make([]byte, size)
+		// Read into buffer of that size.
+		read, err := listxattr(path, &buf[0], size, 0)
+		if err != nil {
+			return nil, &XAttrError{"listxattr", path, "", err}
+		}
+		return nullTermToStrings(buf[:read]), nil
 	}
-	return nullTermToStrings(buf[:read]), nil
+	return []string{}, nil
 }
 
-// Associates name and data together as an attribute of path.
+// Setxattr associates name and data together as an attribute of path.
 func Setxattr(path, name string, data []byte) error {
 	if err := setxattr(path, name, &data[0], len(data), 0, 0); err != nil {
 		return &XAttrError{"setxattr", path, name, err}
@@ -47,7 +50,7 @@ func Setxattr(path, name string, data []byte) error {
 	return nil
 }
 
-// Remove the attribute.
+// Removexattr removes the attribute associated with the given path.
 func Removexattr(path, name string) error {
 	if err := removexattr(path, name, 0); err != nil {
 		return &XAttrError{"removexattr", path, name, err}
diff --git a/vendor/src/github.com/pkg/xattr/xattr_freebsd.go b/vendor/src/github.com/pkg/xattr/xattr_freebsd.go
index 66cf0066b..5173cd2a4 100644
--- a/vendor/src/github.com/pkg/xattr/xattr_freebsd.go
+++ b/vendor/src/github.com/pkg/xattr/xattr_freebsd.go
@@ -10,40 +10,46 @@ const (
 	EXTATTR_NAMESPACE_USER = 1
 )
 
-// Retrieve extended attribute data associated with path.
+// Getxattr retrieves extended attribute data associated with path.
 func Getxattr(path, name string) ([]byte, error) {
 	// find size.
 	size, err := extattr_get_file(path, EXTATTR_NAMESPACE_USER, name, nil, 0)
 	if err != nil {
 		return nil, &XAttrError{"extattr_get_file", path, name, err}
 	}
-	buf := make([]byte, size)
-	// Read into buffer of that size.
-	read, err := extattr_get_file(path, EXTATTR_NAMESPACE_USER, name, &buf[0], size)
-	if err != nil {
-		return nil, &XAttrError{"extattr_get_file", path, name, err}
+	if size > 0 {
+		buf := make([]byte, size)
+		// Read into buffer of that size.
+		read, err := extattr_get_file(path, EXTATTR_NAMESPACE_USER, name, &buf[0], size)
+		if err != nil {
+			return nil, &XAttrError{"extattr_get_file", path, name, err}
+		}
+		return buf[:read], nil
 	}
-	return buf[:read], nil
+	return []byte{}, nil
 }
 
-// Retrieves a list of names of extended attributes associated with the
-// given path in the file system.
+// Listxattr retrieves a list of names of extended attributes associated
+// with the given path in the file system.
 func Listxattr(path string) ([]string, error) {
 	// find size.
 	size, err := extattr_list_file(path, EXTATTR_NAMESPACE_USER, nil, 0)
 	if err != nil {
 		return nil, &XAttrError{"extattr_list_file", path, "", err}
 	}
-	buf := make([]byte, size)
-	// Read into buffer of that size.
-	read, err := extattr_list_file(path, EXTATTR_NAMESPACE_USER, &buf[0], size)
-	if err != nil {
-		return nil, &XAttrError{"extattr_list_file", path, "", err}
+	if size > 0 {
+		buf := make([]byte, size)
+		// Read into buffer of that size.
+		read, err := extattr_list_file(path, EXTATTR_NAMESPACE_USER, &buf[0], size)
+		if err != nil {
+			return nil, &XAttrError{"extattr_list_file", path, "", err}
+		}
+		return attrListToStrings(buf[:read]), nil
 	}
-	return attrListToStrings(buf[:read]), nil
+	return []string{}, nil
 }
 
-// Associates name and data together as an attribute of path.
+// Setxattr associates name and data together as an attribute of path.
 func Setxattr(path, name string, data []byte) error {
 	written, err := extattr_set_file(path, EXTATTR_NAMESPACE_USER, name, &data[0], len(data))
 	if err != nil {
@@ -55,7 +61,7 @@ func Setxattr(path, name string, data []byte) error {
 	return nil
 }
 
-// Remove the attribute.
+// Removexattr removes the attribute associated with the given path.
 func Removexattr(path, name string) error {
 	if err := extattr_delete_file(path, EXTATTR_NAMESPACE_USER, name); err != nil {
 		return &XAttrError{"extattr_delete_file", path, name, err}
@@ -63,7 +69,7 @@ func Removexattr(path, name string) error {
 	return nil
 }
 
-// Convert a sequnce of attribute name entries to a []string.
+// attrListToStrings converts a sequnce of attribute name entries to a []string.
 // Each entry consists of a single byte containing the length
 // of the attribute name, followed by the attribute name.
 // The name is _not_ terminated by NUL.
diff --git a/vendor/src/github.com/pkg/xattr/xattr_linux.go b/vendor/src/github.com/pkg/xattr/xattr_linux.go
index b180d12ab..73973c16a 100644
--- a/vendor/src/github.com/pkg/xattr/xattr_linux.go
+++ b/vendor/src/github.com/pkg/xattr/xattr_linux.go
@@ -4,40 +4,46 @@ package xattr
 
 import "syscall"
 
-// Retrieve extended attribute data associated with path.
+// Getxattr retrieves extended attribute data associated with path.
 func Getxattr(path, name string) ([]byte, error) {
 	// find size.
 	size, err := syscall.Getxattr(path, name, nil)
 	if err != nil {
 		return nil, &XAttrError{"getxattr", path, name, err}
 	}
-	data := make([]byte, size)
-	// Read into buffer of that size.
-	read, err := syscall.Getxattr(path, name, data)
-	if err != nil {
-		return nil, &XAttrError{"getxattr", path, name, err}
+	if size > 0 {
+		data := make([]byte, size)
+		// Read into buffer of that size.
+		read, err := syscall.Getxattr(path, name, data)
+		if err != nil {
+			return nil, &XAttrError{"getxattr", path, name, err}
+		}
+		return data[:read], nil
 	}
-	return data[:read], nil
+	return []byte{}, nil
 }
 
-// Retrieves a list of names of extended attributes associated with the
-// given path in the file system.
+// Listxattr retrieves a list of names of extended attributes associated
+// with the given path in the file system.
 func Listxattr(path string) ([]string, error) {
 	// find size.
 	size, err := syscall.Listxattr(path, nil)
 	if err != nil {
 		return nil, &XAttrError{"listxattr", path, "", err}
 	}
-	buf := make([]byte, size)
-	// Read into buffer of that size.
-	read, err := syscall.Listxattr(path, buf)
-	if err != nil {
-		return nil, &XAttrError{"listxattr", path, "", err}
+	if size > 0 {
+		buf := make([]byte, size)
+		// Read into buffer of that size.
+		read, err := syscall.Listxattr(path, buf)
+		if err != nil {
+			return nil, &XAttrError{"listxattr", path, "", err}
+		}
+		return nullTermToStrings(buf[:read]), nil
 	}
-	return nullTermToStrings(buf[:read]), nil
+	return []string{}, nil
 }
 
-// Associates name and data together as an attribute of path.
+// Setxattr associates name and data together as an attribute of path.
 func Setxattr(path, name string, data []byte) error {
 	if err := syscall.Setxattr(path, name, data, 0); err != nil {
 		return &XAttrError{"setxattr", path, name, err}
@@ -45,7 +51,8 @@ func Setxattr(path, name string, data []byte) error {
 	return nil
 }
 
-// Remove the attribute.
+// Removexattr removes the attribute associated
+// with the given path.
 func Removexattr(path, name string) error {
 	if err := syscall.Removexattr(path, name); err != nil {
 		return &XAttrError{"removexattr", path, name, err}

From 1273c6f3d4542350792122dfe109d17253b85e4f Mon Sep 17 00:00:00 2001
From: Pauline Middelink <middelink@polyware.nl>
Date: Sat, 4 Mar 2017 15:17:44 +0100
Subject: [PATCH 14/28] Display the proper amount of bytes we will be pruning
 from the repo.

---
 src/cmds/restic/cmd_prune.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cmds/restic/cmd_prune.go b/src/cmds/restic/cmd_prune.go
index 1922f35ce..5583b2a15 100644
--- a/src/cmds/restic/cmd_prune.go
+++ b/src/cmds/restic/cmd_prune.go
@@ -184,7 +184,7 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
 		}
 	}
 
-	removeBytes := 0
+	removeBytes := duplicateBytes
 
 	// find packs that are unneeded
 	removePacks := restic.NewIDSet()

From 792b81725e46d9799f11386d904af869b2ec7689 Mon Sep 17 00:00:00 2001
From: Pauline Middelink <middelink@polyware.nl>
Date: Sat, 4 Mar 2017 17:38:34 +0100
Subject: [PATCH 15/28] Add progressbar to repack and blob remove phases of
 prune cmd.

---
 src/cmds/restic/cmd_prune.go         | 27 +++++++++++++++++++--------
 src/restic/repository/repack.go      |  5 ++++-
 src/restic/repository/repack_test.go |  2 +-
 3 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/src/cmds/restic/cmd_prune.go b/src/cmds/restic/cmd_prune.go
index 1922f35ce..64ee09d6c 100644
--- a/src/cmds/restic/cmd_prune.go
+++ b/src/cmds/restic/cmd_prune.go
@@ -217,17 +217,28 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
 	Verbosef("will delete %d packs and rewrite %d packs, this frees %s\n",
 		len(removePacks), len(rewritePacks), formatBytes(uint64(removeBytes)))
 
-	err = repository.Repack(repo, rewritePacks, usedBlobs)
-	if err != nil {
-		return err
+	if len(rewritePacks) != 0 {
+		bar = newProgressMax(!gopts.Quiet, uint64(len(rewritePacks)), "packs rewriten")
+		bar.Start()
+		err = repository.Repack(repo, rewritePacks, usedBlobs, bar)
+		if err != nil {
+			return err
+		}
+		bar.Done()
 	}
 
-	for packID := range removePacks {
-		h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
-		err = repo.Backend().Remove(h)
-		if err != nil {
-			Warnf("unable to remove file %v from the repository\n", packID.Str())
+	if len(removePacks) != 0 {
+		bar = newProgressMax(!gopts.Quiet, uint64(len(removePacks)), "packs deleted")
+		bar.Start()
+		for packID := range removePacks {
+			h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
+			err = repo.Backend().Remove(h)
+			if err != nil {
+				Warnf("unable to remove file %v from the repository\n", packID.Str())
+			}
+			bar.Report(restic.Stat{Blobs: 1})
 		}
+		bar.Done()
 	}
 
 	Verbosef("creating new index\n")
diff --git a/src/restic/repository/repack.go b/src/restic/repository/repack.go
index d0923cc75..7cd1c5fed 100644
--- a/src/restic/repository/repack.go
+++ b/src/restic/repository/repack.go
@@ -18,7 +18,7 @@ import (
 // these packs. Each pack is loaded and the blobs listed in keepBlobs is saved
 // into a new pack. Afterwards, the packs are removed. This operation requires
 // an exclusive lock on the repo.
-func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet) (err error) {
+func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet, p *restic.Progress) (err error) {
 	debug.Log("repacking %d packs while keeping %d blobs", len(packs), len(keepBlobs))
 
 	for packID := range packs {
@@ -118,6 +118,9 @@ func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet
 		if err = os.Remove(tempfile.Name()); err != nil {
 			return errors.Wrap(err, "Remove")
 		}
+		if p != nil {
+			p.Report(restic.Stat{Blobs: 1})
+		}
 	}
 
 	if err := repo.Flush(); err != nil {
diff --git a/src/restic/repository/repack_test.go b/src/restic/repository/repack_test.go
index 619454336..622b3ba52 100644
--- a/src/restic/repository/repack_test.go
+++ b/src/restic/repository/repack_test.go
@@ -132,7 +132,7 @@ func findPacksForBlobs(t *testing.T, repo restic.Repository, blobs restic.BlobSe
 }
 
 func repack(t *testing.T, repo restic.Repository, packs restic.IDSet, blobs restic.BlobSet) {
-	err := repository.Repack(repo, packs, blobs)
+	err := repository.Repack(repo, packs, blobs, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

From 5564c78e53cc054d5c541484c32ca74a269e17dd Mon Sep 17 00:00:00 2001
From: Pauline Middelink <middelink@polyware.nl>
Date: Sat, 4 Mar 2017 17:43:58 +0100
Subject: [PATCH 16/28] English typo: rewriten > rewritten.

---
 src/cmds/restic/cmd_prune.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cmds/restic/cmd_prune.go b/src/cmds/restic/cmd_prune.go
index 64ee09d6c..ecab8196c 100644
--- a/src/cmds/restic/cmd_prune.go
+++ b/src/cmds/restic/cmd_prune.go
@@ -218,7 +218,7 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
 		len(removePacks), len(rewritePacks), formatBytes(uint64(removeBytes)))
 
 	if len(rewritePacks) != 0 {
-		bar = newProgressMax(!gopts.Quiet, uint64(len(rewritePacks)), "packs rewriten")
+		bar = newProgressMax(!gopts.Quiet, uint64(len(rewritePacks)), "packs rewritten")
 		bar.Start()
 		err = repository.Repack(repo, rewritePacks, usedBlobs, bar)
 		if err != nil {

From 7238a3ee89091c5add0277013ba4387b4b94960e Mon Sep 17 00:00:00 2001
From: Pauline Middelink <middelin@google.com>
Date: Sun, 5 Mar 2017 05:24:11 +0100
Subject: [PATCH 17/28] Changed cmd_snapshots to be testable (no more using
 os.Stdout)

---
 src/cmds/restic/cmd_snapshots.go | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/src/cmds/restic/cmd_snapshots.go b/src/cmds/restic/cmd_snapshots.go
index cf803a52b..6a5db6638 100644
--- a/src/cmds/restic/cmd_snapshots.go
+++ b/src/cmds/restic/cmd_snapshots.go
@@ -2,7 +2,7 @@ package main
 
 import (
 	"fmt"
-	"os"
+	"io"
 	"restic/errors"
 	"sort"
 
@@ -64,7 +64,7 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 	for id := range repo.List(restic.SnapshotFile, done) {
 		sn, err := restic.LoadSnapshot(repo, id)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err)
+			Warnf("error loading snapshot %s: %v\n", id, err)
 			continue
 		}
 
@@ -85,19 +85,19 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 	}
 
 	if gopts.JSON {
-		err := printSnapshotsJSON(list)
+		err := printSnapshotsJSON(gopts.stdout, list)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "error printing snapshot: %v\n", err)
+			Warnf("error printing snapshot: %v\n", err)
 		}
 		return nil
 	}
-	printSnapshotsReadable(list)
+	printSnapshotsReadable(gopts.stdout, list)
 
 	return nil
 }
 
 // printSnapshotsReadable prints a text table of the snapshots in list to stdout.
-func printSnapshotsReadable(list []*restic.Snapshot) {
+func printSnapshotsReadable(stdout io.Writer, list []*restic.Snapshot) {
 
 	tab := NewTable()
 	tab.Header = fmt.Sprintf("%-8s  %-19s  %-10s  %-10s  %-3s %s", "ID", "Date", "Host", "Tags", "", "Directory")
@@ -146,9 +146,7 @@ func printSnapshotsReadable(list []*restic.Snapshot) {
 		}
 	}
 
-	tab.Write(os.Stdout)
-
-	return
+	tab.Write(stdout)
 }
 
 // Snapshot helps to print Snaphots as JSON
@@ -159,7 +157,7 @@ type Snapshot struct {
 }
 
 // printSnapshotsJSON writes the JSON representation of list to stdout.
-func printSnapshotsJSON(list []*restic.Snapshot) error {
+func printSnapshotsJSON(stdout io.Writer, list []*restic.Snapshot) error {
 
 	var snapshots []Snapshot
 
@@ -172,6 +170,6 @@ func printSnapshotsJSON(list []*restic.Snapshot) error {
 		snapshots = append(snapshots, k)
 	}
 
-	return json.NewEncoder(os.Stdout).Encode(snapshots)
+	return json.NewEncoder(stdout).Encode(snapshots)
 
 }

From edd5c8b44d52843941fd1f95f8ec74b0e47aeace Mon Sep 17 00:00:00 2001
From: Pauline Middelink <middelin@google.com>
Date: Sun, 5 Mar 2017 05:24:40 +0100
Subject: [PATCH 18/28] Add integration test to make sure cmd_backup adds tags
 when required.

---
 src/cmds/restic/integration_test.go | 53 +++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/src/cmds/restic/integration_test.go b/src/cmds/restic/integration_test.go
index 3c7533ac5..18c103cf1 100644
--- a/src/cmds/restic/integration_test.go
+++ b/src/cmds/restic/integration_test.go
@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"crypto/rand"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -160,6 +161,33 @@ func testRunFind(t testing.TB, gopts GlobalOptions, pattern string) []string {
 	return strings.Split(string(buf.Bytes()), "\n")
 }
 
+func testRunSnapshots(t testing.TB, gopts GlobalOptions) (*Snapshot, map[string]Snapshot) {
+	buf := bytes.NewBuffer(nil)
+	globalOptions.stdout = buf
+	globalOptions.JSON = true
+	defer func() {
+		globalOptions.stdout = os.Stdout
+		globalOptions.JSON = gopts.JSON
+	}()
+
+	opts := SnapshotOptions{}
+
+	OK(t, runSnapshots(opts, globalOptions, []string{}))
+
+	snapshots := []Snapshot{}
+	OK(t, json.Unmarshal(buf.Bytes(), &snapshots))
+
+	var newest *Snapshot
+	snapmap := make(map[string]Snapshot, len(snapshots))
+	for _, sn := range snapshots {
+		snapmap[sn.ID] = sn
+		if newest == nil || sn.Time.After(newest.Time) {
+			newest = &sn
+		}
+	}
+	return newest, snapmap
+}
+
 func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
 	opts := ForgetOptions{}
 	OK(t, runForget(opts, gopts, args))
@@ -602,6 +630,31 @@ func TestIncrementalBackup(t *testing.T) {
 	})
 }
 
+func TestBackupTags(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		testRunInit(t, gopts)
+		SetupTarTestFixture(t, env.testdata, datafile)
+
+		opts := BackupOptions{}
+
+		testRunBackup(t, []string{env.testdata}, opts, gopts)
+		testRunCheck(t, gopts)
+		newest, _ := testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 0,
+			"expected no tags, got %v", newest.Tags)
+
+		opts.Tags = []string{"NL"}
+		testRunBackup(t, []string{env.testdata}, opts, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "NL",
+			"expected one NL tag, got %v", newest.Tags)
+	})
+}
+
 func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string {
 	buf := bytes.NewBuffer(nil)
 

From 45e9f35654cfdec4f6a48b6b37e3e960e90f6875 Mon Sep 17 00:00:00 2001
From: Pauline Middelink <middelin@google.com>
Date: Sun, 5 Mar 2017 05:32:01 +0100
Subject: [PATCH 19/28] Make columns for host and tags size width dynamicly on
 their content.

---
 src/cmds/restic/cmd_snapshots.go | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/cmds/restic/cmd_snapshots.go b/src/cmds/restic/cmd_snapshots.go
index 6a5db6638..634cab0cf 100644
--- a/src/cmds/restic/cmd_snapshots.go
+++ b/src/cmds/restic/cmd_snapshots.go
@@ -99,9 +99,22 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 // printSnapshotsReadable prints a text table of the snapshots in list to stdout.
 func printSnapshotsReadable(stdout io.Writer, list []*restic.Snapshot) {
 
+	// Determine the max widths for host and tag.
+	maxHost, maxTag := 10, 6
+	for _, sn := range list {
+		if len(sn.Hostname) > maxHost {
+			maxHost = len(sn.Hostname)
+		}
+		for _, tag := range sn.Tags {
+			if len(tag) > maxTag {
+				maxTag = len(tag)
+			}
+		}
+	}
+
 	tab := NewTable()
-	tab.Header = fmt.Sprintf("%-8s  %-19s  %-10s  %-10s  %-3s %s", "ID", "Date", "Host", "Tags", "", "Directory")
-	tab.RowFormat = "%-8s  %-19s  %-10s  %-10s  %-3s %s"
+	tab.Header = fmt.Sprintf("%-8s  %-19s  %-*s  %-*s  %-3s %s", "ID", "Date", -maxHost, "Host", -maxTag, "Tags", "", "Directory")
+	tab.RowFormat = fmt.Sprintf("%%-8s  %%-19s  %%%ds  %%%ds  %%-3s %%s", -maxHost, -maxTag)
 
 	for _, sn := range list {
 		if len(sn.Paths) == 0 {

From db085813522d829c44aa324ef242ca11df72faf5 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sun, 5 Mar 2017 16:58:24 +0100
Subject: [PATCH 20/28] Add hint for other backend URI formats

---
 doc/Manual.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/Manual.md b/doc/Manual.md
index 31133a309..3612b6f0e 100644
--- a/doc/Manual.md
+++ b/doc/Manual.md
@@ -143,6 +143,8 @@ Please note that knowledge of your password is required to access the repository
 Losing your password means that your data is irrecoverably lost.
 ```
 
+Other backends like sftp and s3 are [described in a later section](#create-an-sftp-repository) of this document.
+
 Remembering your password is important! If you lose it, you won't be able to
 access data stored in the repository.
 

From f6a258b4a851c1bc31ca5a38573418ccae351c04 Mon Sep 17 00:00:00 2001
From: Pauline Middelink <middelin@google.com>
Date: Sun, 5 Mar 2017 06:20:32 +0100
Subject: [PATCH 21/28] Add `tag`: Manipulate tags on existing snapshots

Add integration testing.
---
 src/cmds/restic/cmd_tag.go          | 189 ++++++++++++++++++++++++++++
 src/cmds/restic/integration_test.go |  56 +++++++++
 2 files changed, 245 insertions(+)
 create mode 100644 src/cmds/restic/cmd_tag.go

diff --git a/src/cmds/restic/cmd_tag.go b/src/cmds/restic/cmd_tag.go
new file mode 100644
index 000000000..9235585f0
--- /dev/null
+++ b/src/cmds/restic/cmd_tag.go
@@ -0,0 +1,189 @@
+package main
+
+import (
+	"github.com/spf13/cobra"
+
+	"restic"
+	"restic/debug"
+	"restic/errors"
+	"restic/repository"
+)
+
+var cmdTag = &cobra.Command{
+	Use:   "tag [flags] [snapshot-ID ...]",
+	Short: "modifies tags on snapshots",
+	Long: `
+The "tag" command allows you to modify tags on exiting snapshots.
+
+You can either set/replace the entire set of tags on a snapshot, or
+add tags to/remove tags from the existing set.
+
+When no snapshot-ID is given, all snapshots matching the host, tag and path filter criteria are modified.
+`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runTag(tagOptions, globalOptions, args)
+	},
+}
+
+// TagOptions bundles all options for the 'tag' command.
+type TagOptions struct {
+	Host       string
+	Paths      []string
+	Tags       []string
+	SetTags    []string
+	AddTags    []string
+	RemoveTags []string
+}
+
+var tagOptions TagOptions
+
+func init() {
+	cmdRoot.AddCommand(cmdTag)
+
+	tagFlags := cmdTag.Flags()
+	tagFlags.StringSliceVar(&tagOptions.SetTags, "set", nil, "`tag` which will replace the existing tags (can be given multiple times)")
+	tagFlags.StringSliceVar(&tagOptions.AddTags, "add", nil, "`tag` which will be added to the existing tags (can be given multiple times)")
+	tagFlags.StringSliceVar(&tagOptions.RemoveTags, "remove", nil, "`tag` which will be removed from the existing tags (can be given multiple times)")
+
+	tagFlags.StringVarP(&tagOptions.Host, "host", "H", "", `only consider snapshots for this host, when no snapshot ID is given`)
+	tagFlags.StringSliceVar(&tagOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given")
+	tagFlags.StringSliceVar(&tagOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
+}
+
+func changeTags(repo *repository.Repository, snapshotID restic.ID, setTags, addTags, removeTags, tags, paths []string, host string) (bool, error) {
+	var changed bool
+
+	sn, err := restic.LoadSnapshot(repo, snapshotID)
+	if err != nil {
+		return false, err
+	}
+	if (host != "" && host != sn.Hostname) || !sn.HasTags(tags) || !restic.SamePaths(sn.Paths, paths) {
+		return false, nil
+	}
+
+	if len(setTags) != 0 {
+		// Setting the tag to an empty string really means no more tags.
+		if len(setTags) == 1 && setTags[0] == "" {
+			setTags = nil
+		}
+		sn.Tags = setTags
+		changed = true
+	} else {
+		for _, add := range addTags {
+			found := false
+			for _, tag := range sn.Tags {
+				if tag == add {
+					found = true
+					break
+				}
+			}
+			if !found {
+				sn.Tags = append(sn.Tags, add)
+				changed = true
+			}
+		}
+		for _, remove := range removeTags {
+			for i, tag := range sn.Tags {
+				if tag == remove {
+					// https://github.com/golang/go/wiki/SliceTricks
+					sn.Tags[i] = sn.Tags[len(sn.Tags)-1]
+					sn.Tags[len(sn.Tags)-1] = ""
+					sn.Tags = sn.Tags[:len(sn.Tags)-1]
+
+					changed = true
+					break
+				}
+			}
+		}
+	}
+
+	if changed {
+		// Save the new snapshot.
+		id, err := repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
+		if err != nil {
+			return false, err
+		}
+
+		debug.Log("new snapshot saved as %v", id.Str())
+
+		if err = repo.Flush(); err != nil {
+			return false, err
+		}
+
+		// Remove the old snapshot.
+		h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+		if err = repo.Backend().Remove(h); err != nil {
+			return false, err
+		}
+
+		debug.Log("old snapshot %v removed", sn.ID())
+	}
+	return changed, nil
+}
+
+func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
+	if len(opts.SetTags) == 0 && len(opts.AddTags) == 0 && len(opts.RemoveTags) == 0 {
+		return errors.Fatal("nothing to do!")
+	}
+	if len(opts.SetTags) != 0 && (len(opts.AddTags) != 0 || len(opts.RemoveTags) != 0) {
+		return errors.Fatal("--set and --add/--remove cannot be given at the same time")
+	}
+
+	repo, err := OpenRepository(gopts)
+	if err != nil {
+		return err
+	}
+
+	if !gopts.NoLock {
+		Verbosef("Create exclusive lock for repository\n")
+		lock, err := lockRepoExclusive(repo)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
+
+	var ids restic.IDs
+	if len(args) != 0 {
+		// When explit snapshot-IDs are given, the filtering does not matter anymore.
+		opts.Host = ""
+		opts.Tags = nil
+		opts.Paths = nil
+
+		// Process all snapshot IDs given as arguments.
+		for _, s := range args {
+			snapshotID, err := restic.FindSnapshot(repo, s)
+			if err != nil {
+				Warnf("could not find a snapshot for ID %q, ignoring: %v\n", s, err)
+				continue
+			}
+			ids = append(ids, snapshotID)
+		}
+		ids = ids.Uniq()
+	} else {
+		// If there were no arguments, just get all snapshots.
+		done := make(chan struct{})
+		defer close(done)
+		for snapshotID := range repo.List(restic.SnapshotFile, done) {
+			ids = append(ids, snapshotID)
+		}
+	}
+
+	changeCnt := 0
+	for _, id := range ids {
+		changed, err := changeTags(repo, id, opts.SetTags, opts.AddTags, opts.RemoveTags, opts.Tags, opts.Paths, opts.Host)
+		if err != nil {
+			Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", id, err)
+			continue
+		}
+		if changed {
+			changeCnt++
+		}
+	}
+	if changeCnt == 0 {
+		Verbosef("No snapshots were modified\n")
+	} else {
+		Verbosef("Modified tags on %v snapshots\n", changeCnt)
+	}
+	return nil
+}
diff --git a/src/cmds/restic/integration_test.go b/src/cmds/restic/integration_test.go
index 18c103cf1..6bb7ed254 100644
--- a/src/cmds/restic/integration_test.go
+++ b/src/cmds/restic/integration_test.go
@@ -655,6 +655,62 @@ func TestBackupTags(t *testing.T) {
 	})
 }
 
+func testRunTag(t testing.TB, opts TagOptions, gopts GlobalOptions) {
+	OK(t, runTag(opts, gopts, []string{}))
+}
+
+func TestTag(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		testRunInit(t, gopts)
+		SetupTarTestFixture(t, env.testdata, datafile)
+
+		testRunBackup(t, []string{env.testdata}, BackupOptions{}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ := testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 0,
+			"expected no tags, got %v", newest.Tags)
+
+		testRunTag(t, TagOptions{SetTags: []string{"NL"}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "NL",
+			"set failed, expected one NL tag, got %v", newest.Tags)
+
+		testRunTag(t, TagOptions{AddTags: []string{"CH"}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 2 && newest.Tags[0] == "NL" && newest.Tags[1] == "CH",
+			"add failed, expected CH,NL tags, got %v", newest.Tags)
+
+		testRunTag(t, TagOptions{RemoveTags: []string{"NL"}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "CH",
+			"remove failed, expected one CH tag, got %v", newest.Tags)
+
+		testRunTag(t, TagOptions{AddTags: []string{"US", "RU"}}, gopts)
+		testRunTag(t, TagOptions{RemoveTags: []string{"CH", "US", "RU"}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 0,
+			"expected no tags, got %v", newest.Tags)
+
+		// Check special case of removing all tags.
+		testRunTag(t, TagOptions{SetTags: []string{""}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 0,
+			"expected no tags, got %v", newest.Tags)
+	})
+}
+
 func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string {
 	buf := bytes.NewBuffer(nil)
 

From 208edaa3d1d50232dc26754692595c68dfb5b6ce Mon Sep 17 00:00:00 2001
From: Pauline Middelink <middelin@google.com>
Date: Sun, 5 Mar 2017 17:43:18 +0100
Subject: [PATCH 22/28] Snapshot: Add `AddTags()` and `RemoveTags()`

Both prevent duplicate tags.
---
 src/cmds/restic/cmd_tag.go | 30 ++++--------------------------
 src/restic/snapshot.go     | 38 ++++++++++++++++++++++++++++++++++++--
 2 files changed, 40 insertions(+), 28 deletions(-)

diff --git a/src/cmds/restic/cmd_tag.go b/src/cmds/restic/cmd_tag.go
index 9235585f0..220e95bc2 100644
--- a/src/cmds/restic/cmd_tag.go
+++ b/src/cmds/restic/cmd_tag.go
@@ -62,38 +62,16 @@ func changeTags(repo *repository.Repository, snapshotID restic.ID, setTags, addT
 	}
 
 	if len(setTags) != 0 {
-		// Setting the tag to an empty string really means no more tags.
+		// Setting the tag to an empty string really means no tags.
 		if len(setTags) == 1 && setTags[0] == "" {
 			setTags = nil
 		}
 		sn.Tags = setTags
 		changed = true
 	} else {
-		for _, add := range addTags {
-			found := false
-			for _, tag := range sn.Tags {
-				if tag == add {
-					found = true
-					break
-				}
-			}
-			if !found {
-				sn.Tags = append(sn.Tags, add)
-				changed = true
-			}
-		}
-		for _, remove := range removeTags {
-			for i, tag := range sn.Tags {
-				if tag == remove {
-					// https://github.com/golang/go/wiki/SliceTricks
-					sn.Tags[i] = sn.Tags[len(sn.Tags)-1]
-					sn.Tags[len(sn.Tags)-1] = ""
-					sn.Tags = sn.Tags[:len(sn.Tags)-1]
-
-					changed = true
-					break
-				}
-			}
+		changed = sn.AddTags(addTags)
+		if sn.RemoveTags(removeTags) {
+			changed = true
 		}
 	}
 
diff --git a/src/restic/snapshot.go b/src/restic/snapshot.go
index 91ffbd558..f57741138 100644
--- a/src/restic/snapshot.go
+++ b/src/restic/snapshot.go
@@ -73,8 +73,7 @@ func LoadAllSnapshots(repo Repository) (snapshots []*Snapshot, err error) {
 
 		snapshots = append(snapshots, sn)
 	}
-
-	return snapshots, nil
+	return
 }
 
 func (sn Snapshot) String() string {
@@ -99,6 +98,41 @@ func (sn *Snapshot) fillUserInfo() error {
 	return err
 }
 
+// AddTags adds the given tags to the snapshots tags, preventing duplicates.
+// It returns true if any changes were made.
+func (sn *Snapshot) AddTags(addTags []string) (changed bool) {
+nextTag:
+	for _, add := range addTags {
+		for _, tag := range sn.Tags {
+			if tag == add {
+				continue nextTag
+			}
+		}
+		sn.Tags = append(sn.Tags, add)
+		changed = true
+	}
+	return
+}
+
+// RemoveTags removes the given tags from the snapshots tags and
+// returns true if any changes were made.
+func (sn *Snapshot) RemoveTags(removeTags []string) (changed bool) {
+	for _, remove := range removeTags {
+		for i, tag := range sn.Tags {
+			if tag == remove {
+				// https://github.com/golang/go/wiki/SliceTricks
+				sn.Tags[i] = sn.Tags[len(sn.Tags)-1]
+				sn.Tags[len(sn.Tags)-1] = ""
+				sn.Tags = sn.Tags[:len(sn.Tags)-1]
+
+				changed = true
+				break
+			}
+		}
+	}
+	return
+}
+
 // HasTags returns true if the snapshot has all the tags.
 func (sn *Snapshot) HasTags(tags []string) bool {
 nextTag:

From 26e266a951e3d1397e86fa920c8373ac29684295 Mon Sep 17 00:00:00 2001
From: Pauline Middelink <middelin@google.com>
Date: Sun, 5 Mar 2017 17:50:11 +0100
Subject: [PATCH 23/28] Fix type of ID field in `cmd_snapshots` type Snapshot

---
 src/cmds/restic/cmd_snapshots.go    | 4 ++--
 src/cmds/restic/integration_test.go | 9 ++++-----
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/src/cmds/restic/cmd_snapshots.go b/src/cmds/restic/cmd_snapshots.go
index 634cab0cf..5e3db0c5c 100644
--- a/src/cmds/restic/cmd_snapshots.go
+++ b/src/cmds/restic/cmd_snapshots.go
@@ -166,7 +166,7 @@ func printSnapshotsReadable(stdout io.Writer, list []*restic.Snapshot) {
 type Snapshot struct {
 	*restic.Snapshot
 
-	ID string `json:"id"`
+	ID *restic.ID `json:"id"`
 }
 
 // printSnapshotsJSON writes the JSON representation of list to stdout.
@@ -178,7 +178,7 @@ func printSnapshotsJSON(stdout io.Writer, list []*restic.Snapshot) error {
 
 		k := Snapshot{
 			Snapshot: sn,
-			ID:       sn.ID().String(),
+			ID:       sn.ID(),
 		}
 		snapshots = append(snapshots, k)
 	}
diff --git a/src/cmds/restic/integration_test.go b/src/cmds/restic/integration_test.go
index 6bb7ed254..a4ad49ba9 100644
--- a/src/cmds/restic/integration_test.go
+++ b/src/cmds/restic/integration_test.go
@@ -161,7 +161,7 @@ func testRunFind(t testing.TB, gopts GlobalOptions, pattern string) []string {
 	return strings.Split(string(buf.Bytes()), "\n")
 }
 
-func testRunSnapshots(t testing.TB, gopts GlobalOptions) (*Snapshot, map[string]Snapshot) {
+func testRunSnapshots(t testing.TB, gopts GlobalOptions) (newest *Snapshot, snapmap map[restic.ID]Snapshot) {
 	buf := bytes.NewBuffer(nil)
 	globalOptions.stdout = buf
 	globalOptions.JSON = true
@@ -177,15 +177,14 @@ func testRunSnapshots(t testing.TB, gopts GlobalOptions) (*Snapshot, map[string]
 	snapshots := []Snapshot{}
 	OK(t, json.Unmarshal(buf.Bytes(), &snapshots))
 
-	var newest *Snapshot
-	snapmap := make(map[string]Snapshot, len(snapshots))
+	snapmap = make(map[restic.ID]Snapshot, len(snapshots))
 	for _, sn := range snapshots {
-		snapmap[sn.ID] = sn
+		snapmap[*sn.ID] = sn
 		if newest == nil || sn.Time.After(newest.Time) {
 			newest = &sn
 		}
 	}
-	return newest, snapmap
+	return
 }
 
 func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {

From 1fa2313aefeb0f307afc77be8099073a69d45e5c Mon Sep 17 00:00:00 2001
From: Pauline Middelink <middelin@google.com>
Date: Sun, 5 Mar 2017 17:51:57 +0100
Subject: [PATCH 24/28] Snapshot: Add Original ID

The Original ID is used when the snapshot is modified (e.g. by `tag`
command). Adjust integration testing to assert correctness.
---
 src/cmds/restic/cmd_tag.go          |  5 +++++
 src/cmds/restic/integration_test.go | 18 ++++++++++++++++++
 src/restic/snapshot.go              |  1 +
 3 files changed, 24 insertions(+)

diff --git a/src/cmds/restic/cmd_tag.go b/src/cmds/restic/cmd_tag.go
index 220e95bc2..257320352 100644
--- a/src/cmds/restic/cmd_tag.go
+++ b/src/cmds/restic/cmd_tag.go
@@ -76,6 +76,11 @@ func changeTags(repo *repository.Repository, snapshotID restic.ID, setTags, addT
 	}
 
 	if changed {
+		// Retain the original snapshot id over all tag changes.
+		if sn.Original == nil {
+			sn.Original = sn.ID()
+		}
+
 		// Save the new snapshot.
 		id, err := repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
 		if err != nil {
diff --git a/src/cmds/restic/integration_test.go b/src/cmds/restic/integration_test.go
index a4ad49ba9..9d7ea55fe 100644
--- a/src/cmds/restic/integration_test.go
+++ b/src/cmds/restic/integration_test.go
@@ -670,6 +670,9 @@ func TestTag(t *testing.T) {
 		Assert(t, newest != nil, "expected a new backup, got nil")
 		Assert(t, len(newest.Tags) == 0,
 			"expected no tags, got %v", newest.Tags)
+		Assert(t, newest.Original == nil,
+			"expected original ID to be nil, got %v", newest.Original)
+		originalID := *newest.ID
 
 		testRunTag(t, TagOptions{SetTags: []string{"NL"}}, gopts)
 		testRunCheck(t, gopts)
@@ -677,6 +680,9 @@ func TestTag(t *testing.T) {
 		Assert(t, newest != nil, "expected a new backup, got nil")
 		Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "NL",
 			"set failed, expected one NL tag, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
 
 		testRunTag(t, TagOptions{AddTags: []string{"CH"}}, gopts)
 		testRunCheck(t, gopts)
@@ -684,6 +690,9 @@ func TestTag(t *testing.T) {
 		Assert(t, newest != nil, "expected a new backup, got nil")
 		Assert(t, len(newest.Tags) == 2 && newest.Tags[0] == "NL" && newest.Tags[1] == "CH",
 			"add failed, expected CH,NL tags, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
 
 		testRunTag(t, TagOptions{RemoveTags: []string{"NL"}}, gopts)
 		testRunCheck(t, gopts)
@@ -691,6 +700,9 @@ func TestTag(t *testing.T) {
 		Assert(t, newest != nil, "expected a new backup, got nil")
 		Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "CH",
 			"remove failed, expected one CH tag, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
 
 		testRunTag(t, TagOptions{AddTags: []string{"US", "RU"}}, gopts)
 		testRunTag(t, TagOptions{RemoveTags: []string{"CH", "US", "RU"}}, gopts)
@@ -699,6 +711,9 @@ func TestTag(t *testing.T) {
 		Assert(t, newest != nil, "expected a new backup, got nil")
 		Assert(t, len(newest.Tags) == 0,
 			"expected no tags, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
 
 		// Check special case of removing all tags.
 		testRunTag(t, TagOptions{SetTags: []string{""}}, gopts)
@@ -707,6 +722,9 @@ func TestTag(t *testing.T) {
 		Assert(t, newest != nil, "expected a new backup, got nil")
 		Assert(t, len(newest.Tags) == 0,
 			"expected no tags, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
 	})
 }
 
diff --git a/src/restic/snapshot.go b/src/restic/snapshot.go
index f57741138..68f5e6878 100644
--- a/src/restic/snapshot.go
+++ b/src/restic/snapshot.go
@@ -21,6 +21,7 @@ type Snapshot struct {
 	GID      uint32    `json:"gid,omitempty"`
 	Excludes []string  `json:"excludes,omitempty"`
 	Tags     []string  `json:"tags,omitempty"`
+	Original *ID       `json:"original,omitempty"`
 
 	id *ID // plaintext ID, used during restore
 }

From be15a9261ad101949eb93d68d18c51fcee55ef03 Mon Sep 17 00:00:00 2001
From: Pauline Middelink <middelin@google.com>
Date: Sun, 5 Mar 2017 19:06:06 +0100
Subject: [PATCH 25/28] Add design and user documentation for the `restic tag`
 command

---
 doc/Design.md | 16 +++++++++++++---
 doc/Manual.md | 29 +++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/doc/Design.md b/doc/Design.md
index 117554d2b..e244245a2 100644
--- a/doc/Design.md
+++ b/doc/Design.md
@@ -285,7 +285,7 @@ This way, the password can be changed without having to re-encrypt all data.
 Snapshots
 ---------
 
-A snapshots represents a directory with all files and sub-directories at a
+A snapshot represents a directory with all files and sub-directories at a
 given point in time. For each backup that is made, a new snapshot is created. A
 snapshot is a JSON document that is stored in an encrypted file below the
 directory `snapshots` in the repository. The filename is the storage ID. This
@@ -304,12 +304,22 @@ enter password for repository:
   "hostname": "kasimir",
   "username": "fd0",
   "uid": 1000,
-  "gid": 100
+  "gid": 100,
+  "tags": [
+    "NL"
+  ],
+  "original": "251c2e5841355f743f9d4ffd3260bee765acee40a6229857e32b60446991b837"
 }
 ```
 
 Here it can be seen that this snapshot represents the contents of the directory
-`/tmp/testdata`. The most important field is `tree`.
+`/tmp/testdata` after its tags were changed to "NL". The most important field
+is `tree`.
+
+Another important field is `original`, if any modification is made to the
+snapshot, say because its tags changed, its SHA-256 hash changes and therefore
+the original snapshot id is lost. Retaining a stable id is especially important
+for caching.
 
 All content within a restic repository is referenced according to its SHA-256
 hash. Before saving, each file is split into variable sized Blobs of data. The
diff --git a/doc/Manual.md b/doc/Manual.md
index 3612b6f0e..fe52735d9 100644
--- a/doc/Manual.md
+++ b/doc/Manual.md
@@ -73,6 +73,7 @@ Available Commands:
   rebuild-index build a new index file
   restore       extract the data from a snapshot
   snapshots     list all snapshots
+  tag           modifies tags on snapshots
   unlock        remove locks other processes created
   version       Print version information
 
@@ -394,6 +395,34 @@ enter password for repository:
 *eb78040b    username    kasimir   2015-08-12 13:29:57
 ```
 
+# Manage tags
+
+Managing tags on snapshots is simple. The existing set of tags can be either
+replaced completely, added to or removed from. The result is directly visible
+in the `snapshots` command.
+
+```console
+$ restic -r /tmp/backup tag --set NL,CH 590c8fc8
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+```
+Note the snapshot ID has changed, so between each change we need to look up
+the new ID of the snapshot. But there is an even better way, the `tag` command
+accepts `--tag`, so we can filter snapshots based on the tag we just added.
+```console
+$ restic -r /tmp/backup tag --tag NL --remove CH
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+$ restic -r /tmp/backup tag --tag NL --add UK
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+$ restic -r /tmp/backup tag --tag NL --remove NL
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+$ restic -r /tmp/backup tag --tag NL --add SOMETHING
+No snapshots were modified
+```
+
 # Check integrity and consistency
 
 Imagine your repository is saved on a server that has a faulty hard drive, or

From 07695b3622c9dd3e67babea59d25ef6a5731cbae Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sun, 5 Mar 2017 20:12:25 +0100
Subject: [PATCH 26/28] Documentation fixes

---
 doc/Design.md | 38 +++++++++++++++++++++++++++++---------
 doc/Manual.md | 19 +++++++++++++++----
 2 files changed, 44 insertions(+), 13 deletions(-)

diff --git a/doc/Design.md b/doc/Design.md
index e244245a2..52a228a93 100644
--- a/doc/Design.md
+++ b/doc/Design.md
@@ -295,7 +295,7 @@ The command `restic cat snapshot` can be used as follows to decrypt and
 pretty-print the contents of a snapshot file:
 
 ```console
-$ restic -r /tmp/restic-repo cat snapshot 22a5af1b
+$ restic -r /tmp/restic-repo cat snapshot 251c2e58
 enter password for repository:
 {
   "time": "2015-01-02T18:10:50.895208559+01:00",
@@ -307,19 +307,39 @@ enter password for repository:
   "gid": 100,
   "tags": [
     "NL"
+  ]
+}
+```
+
+Here it can be seen that this snapshot represents the contents of the directory
+`/tmp/testdata`. The most important field is `tree`. When the meta data (e.g.
+the tags) of a snapshot change, the snapshot needs to be re-encrypted and saved.
+This will change the storage ID, so in order to relate these seemingly
+different snapshots, a field `original` is introduced which contains the ID of
+the original snapshot, e.g. after adding the tag `DE` to the snapshot above it
+becomes:
+
+```console
+$ restic -r /tmp/restic-repo cat snapshot 22a5af1b
+enter password for repository:
+{
+  "time": "2015-01-02T18:10:50.895208559+01:00",
+  "tree": "2da81727b6585232894cfbb8f8bdab8d1eccd3d8f7c92bc934d62e62e618ffdf",
+  "dir": "/tmp/testdata",
+  "hostname": "kasimir",
+  "username": "fd0",
+  "uid": 1000,
+  "gid": 100,
+  "tags": [
+    "NL",
+    "DE"
   ],
   "original": "251c2e5841355f743f9d4ffd3260bee765acee40a6229857e32b60446991b837"
 }
 ```
 
-Here it can be seen that this snapshot represents the contents of the directory
-`/tmp/testdata` after its tags were changed to "NL". The most important field
-is `tree`.
-
-Another important field is `original`, if any modification is made to the
-snapshot, say because its tags changed, its SHA-256 hash changes and therefore
-the original snapshot id is lost. Retaining a stable id is especially important
-for caching.
+Once introduced, the `original` field is not modified when the snapshot's meta
+data is changed again.
 
 All content within a restic repository is referenced according to its SHA-256
 hash. Before saving, each file is split into variable sized Blobs of data. The
diff --git a/doc/Manual.md b/doc/Manual.md
index fe52735d9..513ef612c 100644
--- a/doc/Manual.md
+++ b/doc/Manual.md
@@ -397,28 +397,39 @@ enter password for repository:
 
 # Manage tags
 
-Managing tags on snapshots is simple. The existing set of tags can be either
-replaced completely, added to or removed from. The result is directly visible
-in the `snapshots` command.
+Managing tags on snapshots is done with the `tag` command. The existing set of
+tags can be replaced completely, tags can be added to removed. The result is
+directly visible in the `snapshots` command.
+
+Let's say we want to tag snapshot `590c8fc8` with the tags `NL` and `CH` and
+remove all other tags that may be present, the following command does that:
 
 ```console
 $ restic -r /tmp/backup tag --set NL,CH 590c8fc8
 Create exclusive lock for repository
 Modified tags on 1 snapshots
 ```
+
 Note the snapshot ID has changed, so between each change we need to look up
 the new ID of the snapshot. But there is an even better way, the `tag` command
-accepts `--tag`, so we can filter snapshots based on the tag we just added.
+accepts `--tag` for a filter, so we can filter snapshots based on the tag we
+just added.
+
+So we can add and remove tags incrementally like this:
+
 ```console
 $ restic -r /tmp/backup tag --tag NL --remove CH
 Create exclusive lock for repository
 Modified tags on 1 snapshots
+
 $ restic -r /tmp/backup tag --tag NL --add UK
 Create exclusive lock for repository
 Modified tags on 1 snapshots
+
 $ restic -r /tmp/backup tag --tag NL --remove NL
 Create exclusive lock for repository
 Modified tags on 1 snapshots
+
 $ restic -r /tmp/backup tag --tag NL --add SOMETHING
 No snapshots were modified
 ```

From e1c828be3e00cc2daf7d0dc42eb2f910ece45e4f Mon Sep 17 00:00:00 2001
From: Pauline Middelink <middelin@google.com>
Date: Mon, 6 Mar 2017 02:21:58 +0100
Subject: [PATCH 27/28] Fix SamePaths() and make it into a receiver function

Add `HasPath(paths []string) bool` to Snapshot for testing if the
snapshot has at least the paths given to the function.

Reimplemented SamePaths(paths []string) so it does what the name implies,
compare if all given paths are in the snapshot.
---
 src/cmds/restic/cmd_snapshots.go |  2 +-
 src/cmds/restic/cmd_tag.go       |  2 +-
 src/restic/snapshot.go           | 36 +++++++++++++++++---------------
 3 files changed, 21 insertions(+), 19 deletions(-)

diff --git a/src/cmds/restic/cmd_snapshots.go b/src/cmds/restic/cmd_snapshots.go
index 5e3db0c5c..a78342994 100644
--- a/src/cmds/restic/cmd_snapshots.go
+++ b/src/cmds/restic/cmd_snapshots.go
@@ -68,7 +68,7 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 			continue
 		}
 
-		if restic.SamePaths(sn.Paths, opts.Paths) && (opts.Host == "" || opts.Host == sn.Hostname) {
+		if (opts.Host == "" || opts.Host == sn.Hostname) && sn.HasPaths(opts.Paths) {
 			pos := sort.Search(len(list), func(i int) bool {
 				return list[i].Time.After(sn.Time)
 			})
diff --git a/src/cmds/restic/cmd_tag.go b/src/cmds/restic/cmd_tag.go
index 257320352..1d8d5ebd2 100644
--- a/src/cmds/restic/cmd_tag.go
+++ b/src/cmds/restic/cmd_tag.go
@@ -57,7 +57,7 @@ func changeTags(repo *repository.Repository, snapshotID restic.ID, setTags, addT
 	if err != nil {
 		return false, err
 	}
-	if (host != "" && host != sn.Hostname) || !sn.HasTags(tags) || !restic.SamePaths(sn.Paths, paths) {
+	if (host != "" && host != sn.Hostname) || !sn.HasTags(tags) || !sn.HasPaths(paths) {
 		return false, nil
 	}
 
diff --git a/src/restic/snapshot.go b/src/restic/snapshot.go
index 68f5e6878..ed89a60ca 100644
--- a/src/restic/snapshot.go
+++ b/src/restic/snapshot.go
@@ -134,7 +134,7 @@ func (sn *Snapshot) RemoveTags(removeTags []string) (changed bool) {
 	return
 }
 
-// HasTags returns true if the snapshot has all the tags.
+// HasTags returns true if the snapshot has at least all of tags.
 func (sn *Snapshot) HasTags(tags []string) bool {
 nextTag:
 	for _, tag := range tags {
@@ -150,28 +150,30 @@ nextTag:
 	return true
 }
 
-// SamePaths compares the Snapshot's paths and provided paths are exactly the same
-func SamePaths(expected, actual []string) bool {
-	if len(expected) == 0 || len(actual) == 0 {
-		return true
-	}
-
-	for i := range expected {
-		found := false
-		for j := range actual {
-			if expected[i] == actual[j] {
-				found = true
-				break
+// HasPaths returns true if the snapshot has at least all of paths.
+func (sn *Snapshot) HasPaths(paths []string) bool {
+nextPath:
+	for _, path := range paths {
+		for _, snPath := range sn.Paths {
+			if path == snPath {
+				continue nextPath
 			}
 		}
-		if !found {
-			return false
-		}
+
+		return false
 	}
 
 	return true
 }
 
+// SamePaths returns true if the snapshot matches the entire paths set
+func (sn *Snapshot) SamePaths(paths []string) bool {
+	if len(sn.Paths) != len(paths) {
+		return false
+	}
+	return sn.HasPaths(paths)
+}
+
 // ErrNoSnapshotFound is returned when no snapshot for the given criteria could be found.
 var ErrNoSnapshotFound = errors.New("no snapshot found")
 
@@ -188,7 +190,7 @@ func FindLatestSnapshot(repo Repository, targets []string, hostname string) (ID,
 		if err != nil {
 			return ID{}, errors.Errorf("Error listing snapshot: %v", err)
 		}
-		if snapshot.Time.After(latest) && SamePaths(snapshot.Paths, targets) && (hostname == "" || hostname == snapshot.Hostname) {
+		if snapshot.Time.After(latest) && snapshot.HasPaths(targets) && (hostname == "" || hostname == snapshot.Hostname) {
 			latest = snapshot.Time
 			latestID = snapshotID
 			found = true

From 354e8ffb8201d5a5e1c5b7756c27e265e8b2d224 Mon Sep 17 00:00:00 2001
From: Pauline Middelink <middelin@google.com>
Date: Mon, 6 Mar 2017 02:49:15 +0100
Subject: [PATCH 28/28] Fix layout issue in cmd_snapshot "ascii art"
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The layouter does not account for multi tags when determining the
need for ascii art.

36fd8178  2017-03-03 21:35:04  abuseio.polyware.nl    NL          /
                                                      A       └──

vs

36fd8178  2017-03-03 21:35:04  abuseio.polyware.nl    NL      ┌── /
                                                      A       └──
---
 src/cmds/restic/cmd_snapshots.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/cmds/restic/cmd_snapshots.go b/src/cmds/restic/cmd_snapshots.go
index 5e3db0c5c..b6ef3bd2f 100644
--- a/src/cmds/restic/cmd_snapshots.go
+++ b/src/cmds/restic/cmd_snapshots.go
@@ -127,6 +127,9 @@ func printSnapshotsReadable(stdout io.Writer, list []*restic.Snapshot) {
 		}
 
 		rows := len(sn.Paths)
+		if rows < len(sn.Tags) {
+			rows = len(sn.Tags)
+		}
 
 		treeElement := "   "
 		if rows != 1 {