From ffcb0155819aacada563c3afe6b943f4145c39c7 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 5 Apr 2017 20:42:15 +0200 Subject: [PATCH 1/6] Add new field DeviceID and tests --- src/restic/node.go | 6 ++- src/restic/node_test.go | 109 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/src/restic/node.go b/src/restic/node.go index b85bfee80..0be95949d 100644 --- a/src/restic/node.go +++ b/src/restic/node.go @@ -37,11 +37,12 @@ type Node struct { User string `json:"user,omitempty"` Group string `json:"group,omitempty"` Inode uint64 `json:"inode,omitempty"` + DeviceID uint64 `json:"device_id,omitempty"` // device id of the file, stat.st_dev Size uint64 `json:"size,omitempty"` Links uint64 `json:"links,omitempty"` LinkTarget string `json:"linktarget,omitempty"` ExtendedAttributes []ExtendedAttribute `json:"extended_attributes,omitempty"` - Device uint64 `json:"device,omitempty"` + Device uint64 `json:"device,omitempty"` // in case of Type == "dev", stat.st_rdev Content IDs `json:"content"` Subtree *ID `json:"subtree,omitempty"` @@ -376,6 +377,9 @@ func (node Node) Equals(other Node) bool { if node.Inode != other.Inode { return false } + if node.DeviceID != other.DeviceID { + return false + } if node.Size != other.Size { return false } diff --git a/src/restic/node_test.go b/src/restic/node_test.go index 16414150f..7668a43f0 100644 --- a/src/restic/node_test.go +++ b/src/restic/node_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "runtime" + "syscall" "testing" "time" @@ -241,3 +242,111 @@ func AssertFsTimeEqual(t *testing.T, label string, nodeType string, t1 time.Time Assert(t, equal, "%s: %s doesn't match (%v != %v)", label, nodeType, t1, t2) } + +func stat(t testing.TB, filename string) (fi os.FileInfo, ok bool) { + fi, err := os.Lstat(filename) + if err != nil && os.IsNotExist(err) { + return fi, false + } + + if err != nil { + t.Fatal(err) + } + + return fi, true +} + +func checkFile(t testing.TB, stat *syscall.Stat_t, node *restic.Node) { + if uint32(node.Mode.Perm()) != stat.Mode&0777 { + t.Errorf("Mode does not match, want %v, got %v", stat.Mode&0777, node.Mode) + } + + if node.Inode != stat.Ino { + t.Errorf("Inode does not match, want %v, got %v", stat.Ino, node.Inode) + } + + if node.DeviceID != stat.Dev { + t.Errorf("Dev does not match, want %v, got %v", stat.Dev, node.DeviceID) + } + + if node.Size != uint64(stat.Size) { + t.Errorf("Size does not match, want %v, got %v", stat.Size, node.Size) + } + + if node.Links != stat.Nlink { + t.Errorf("Links does not match, want %v, got %v", stat.Nlink, node.Links) + } + + if node.ModTime != time.Unix(stat.Mtim.Unix()) { + t.Errorf("ModTime does not match, want %v, got %v", time.Unix(stat.Mtim.Unix()), node.ModTime) + } + + if node.ChangeTime != time.Unix(stat.Ctim.Unix()) { + t.Errorf("ChangeTime does not match, want %v, got %v", time.Unix(stat.Ctim.Unix()), node.ChangeTime) + } + + if node.AccessTime != time.Unix(stat.Atim.Unix()) { + t.Errorf("AccessTime does not match, want %v, got %v", time.Unix(stat.Atim.Unix()), node.AccessTime) + } + + if node.UID != stat.Uid { + t.Errorf("UID does not match, want %v, got %v", stat.Uid, node.UID) + } + + if node.GID != stat.Gid { + t.Errorf("UID does not match, want %v, got %v", stat.Gid, node.GID) + } +} + +func checkDevice(t testing.TB, stat *syscall.Stat_t, node *restic.Node) { + if node.Device != stat.Rdev { + t.Errorf("Rdev does not match, want %v, got %v", stat.Rdev, node.Device) + } +} + +func TestNodeFromFileInfo(t *testing.T) { + var tests = []struct { + filename string + canSkip bool + }{ + {"node_test.go", false}, + {"/dev/null", true}, + {"/dev/sda", true}, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + fi, found := stat(t, test.filename) + if !found && test.canSkip { + t.Skipf("%v not found in filesystem") + return + } + + if fi.Sys() == nil { + t.Skip("fi.Sys() is nil") + return + } + + s, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + t.Skip("fi type is %T, not stat_t", fi.Sys()) + return + } + + node, err := restic.NodeFromFileInfo(test.filename, fi) + if err != nil { + t.Fatal(err) + } + + switch node.Type { + case "file": + checkFile(t, s, node) + case "dev", "chardev": + checkFile(t, s, node) + checkDevice(t, s, node) + default: + t.Fatalf("invalid node type %q", node.Type) + } + }) + } +} From 6f1b03415c55100bf6d3409a64d51277a4a61c26 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 5 Apr 2017 20:45:24 +0200 Subject: [PATCH 2/6] Fix hardlinks --- src/restic/node.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/restic/node.go b/src/restic/node.go index 0be95949d..25e51aa48 100644 --- a/src/restic/node.go +++ b/src/restic/node.go @@ -229,11 +229,11 @@ func (node Node) createDirAt(path string) error { } func (node Node) createFileAt(path string, repo Repository, idx *HardlinkIndex) error { - if node.Links > 1 && idx.Has(node.Inode, node.Device) { + if node.Links > 1 && idx.Has(node.Inode, node.DeviceID) { if err := fs.Remove(path); !os.IsNotExist(err) { return errors.Wrap(err, "RemoveCreateHardlink") } - err := fs.Link(idx.GetFilename(node.Inode, node.Device), path) + err := fs.Link(idx.GetFilename(node.Inode, node.DeviceID), path) if err != nil { return errors.Wrap(err, "CreateHardlink") } @@ -272,7 +272,7 @@ func (node Node) createFileAt(path string, repo Repository, idx *HardlinkIndex) } if node.Links > 1 { - idx.Add(node.Inode, node.Device, path) + idx.Add(node.Inode, node.DeviceID, path) } return nil @@ -572,7 +572,7 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error { } node.Inode = uint64(stat.ino()) - node.Device = uint64(stat.dev()) + node.DeviceID = uint64(stat.dev()) node.fillTimes(stat) From c195139d313736f2e0e93c31ef38fd72450c92b8 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 5 Apr 2017 20:51:26 +0200 Subject: [PATCH 3/6] Only run tests on unix --- src/restic/node_test.go | 109 -------------------------------- src/restic/node_unix_test.go | 119 +++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 109 deletions(-) create mode 100644 src/restic/node_unix_test.go diff --git a/src/restic/node_test.go b/src/restic/node_test.go index 7668a43f0..16414150f 100644 --- a/src/restic/node_test.go +++ b/src/restic/node_test.go @@ -5,7 +5,6 @@ import ( "os" "path/filepath" "runtime" - "syscall" "testing" "time" @@ -242,111 +241,3 @@ func AssertFsTimeEqual(t *testing.T, label string, nodeType string, t1 time.Time Assert(t, equal, "%s: %s doesn't match (%v != %v)", label, nodeType, t1, t2) } - -func stat(t testing.TB, filename string) (fi os.FileInfo, ok bool) { - fi, err := os.Lstat(filename) - if err != nil && os.IsNotExist(err) { - return fi, false - } - - if err != nil { - t.Fatal(err) - } - - return fi, true -} - -func checkFile(t testing.TB, stat *syscall.Stat_t, node *restic.Node) { - if uint32(node.Mode.Perm()) != stat.Mode&0777 { - t.Errorf("Mode does not match, want %v, got %v", stat.Mode&0777, node.Mode) - } - - if node.Inode != stat.Ino { - t.Errorf("Inode does not match, want %v, got %v", stat.Ino, node.Inode) - } - - if node.DeviceID != stat.Dev { - t.Errorf("Dev does not match, want %v, got %v", stat.Dev, node.DeviceID) - } - - if node.Size != uint64(stat.Size) { - t.Errorf("Size does not match, want %v, got %v", stat.Size, node.Size) - } - - if node.Links != stat.Nlink { - t.Errorf("Links does not match, want %v, got %v", stat.Nlink, node.Links) - } - - if node.ModTime != time.Unix(stat.Mtim.Unix()) { - t.Errorf("ModTime does not match, want %v, got %v", time.Unix(stat.Mtim.Unix()), node.ModTime) - } - - if node.ChangeTime != time.Unix(stat.Ctim.Unix()) { - t.Errorf("ChangeTime does not match, want %v, got %v", time.Unix(stat.Ctim.Unix()), node.ChangeTime) - } - - if node.AccessTime != time.Unix(stat.Atim.Unix()) { - t.Errorf("AccessTime does not match, want %v, got %v", time.Unix(stat.Atim.Unix()), node.AccessTime) - } - - if node.UID != stat.Uid { - t.Errorf("UID does not match, want %v, got %v", stat.Uid, node.UID) - } - - if node.GID != stat.Gid { - t.Errorf("UID does not match, want %v, got %v", stat.Gid, node.GID) - } -} - -func checkDevice(t testing.TB, stat *syscall.Stat_t, node *restic.Node) { - if node.Device != stat.Rdev { - t.Errorf("Rdev does not match, want %v, got %v", stat.Rdev, node.Device) - } -} - -func TestNodeFromFileInfo(t *testing.T) { - var tests = []struct { - filename string - canSkip bool - }{ - {"node_test.go", false}, - {"/dev/null", true}, - {"/dev/sda", true}, - } - - for _, test := range tests { - t.Run("", func(t *testing.T) { - fi, found := stat(t, test.filename) - if !found && test.canSkip { - t.Skipf("%v not found in filesystem") - return - } - - if fi.Sys() == nil { - t.Skip("fi.Sys() is nil") - return - } - - s, ok := fi.Sys().(*syscall.Stat_t) - if !ok { - t.Skip("fi type is %T, not stat_t", fi.Sys()) - return - } - - node, err := restic.NodeFromFileInfo(test.filename, fi) - if err != nil { - t.Fatal(err) - } - - switch node.Type { - case "file": - checkFile(t, s, node) - case "dev", "chardev": - checkFile(t, s, node) - checkDevice(t, s, node) - default: - t.Fatalf("invalid node type %q", node.Type) - } - }) - } -} diff --git a/src/restic/node_unix_test.go b/src/restic/node_unix_test.go new file mode 100644 index 000000000..a581b3466 --- /dev/null +++ b/src/restic/node_unix_test.go @@ -0,0 +1,119 @@ +// +build !windows + +package restic_test + +import ( + "os" + "restic" + "syscall" + "testing" + "time" +) + +func stat(t testing.TB, filename string) (fi os.FileInfo, ok bool) { + fi, err := os.Lstat(filename) + if err != nil && os.IsNotExist(err) { + return fi, false + } + + if err != nil { + t.Fatal(err) + } + + return fi, true +} + +func checkFile(t testing.TB, stat *syscall.Stat_t, node *restic.Node) { + if uint32(node.Mode.Perm()) != stat.Mode&0777 { + t.Errorf("Mode does not match, want %v, got %v", stat.Mode&0777, node.Mode) + } + + if node.Inode != stat.Ino { + t.Errorf("Inode does not match, want %v, got %v", stat.Ino, node.Inode) + } + + if node.DeviceID != stat.Dev { + t.Errorf("Dev does not match, want %v, got %v", stat.Dev, node.DeviceID) + } + + if node.Size != uint64(stat.Size) { + t.Errorf("Size does not match, want %v, got %v", stat.Size, node.Size) + } + + if node.Links != stat.Nlink { + t.Errorf("Links does not match, want %v, got %v", stat.Nlink, node.Links) + } + + if node.ModTime != time.Unix(stat.Mtim.Unix()) { + t.Errorf("ModTime does not match, want %v, got %v", time.Unix(stat.Mtim.Unix()), node.ModTime) + } + + if node.ChangeTime != time.Unix(stat.Ctim.Unix()) { + t.Errorf("ChangeTime does not match, want %v, got %v", time.Unix(stat.Ctim.Unix()), node.ChangeTime) + } + + if node.AccessTime != time.Unix(stat.Atim.Unix()) { + t.Errorf("AccessTime does not match, want %v, got %v", time.Unix(stat.Atim.Unix()), node.AccessTime) + } + + if node.UID != stat.Uid { + t.Errorf("UID does not match, want %v, got %v", stat.Uid, node.UID) + } + + if node.GID != stat.Gid { + t.Errorf("UID does not match, want %v, got %v", stat.Gid, node.GID) + } +} + +func checkDevice(t testing.TB, stat *syscall.Stat_t, node *restic.Node) { + if node.Device != stat.Rdev { + t.Errorf("Rdev does not match, want %v, got %v", stat.Rdev, node.Device) + } +} + +func TestNodeFromFileInfo(t *testing.T) { + var tests = []struct { + filename string + canSkip bool + }{ + {"node_test.go", false}, + {"/dev/null", true}, + {"/dev/sda", true}, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + fi, found := stat(t, test.filename) + if !found && test.canSkip { + t.Skipf("%v not found in filesystem") + return + } + + if fi.Sys() == nil { + t.Skip("fi.Sys() is nil") + return + } + + s, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + t.Skip("fi type is %T, not stat_t", fi.Sys()) + return + } + + node, err := restic.NodeFromFileInfo(test.filename, fi) + if err != nil { + t.Fatal(err) + } + + switch node.Type { + case "file": + checkFile(t, s, node) + case "dev", "chardev": + checkFile(t, s, node) + checkDevice(t, s, node) + default: + t.Fatalf("invalid node type %q", node.Type) + } + }) + } +} From 280028290e600cb9d60a0f3a9a9036834684710a Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 5 Apr 2017 21:35:29 +0200 Subject: [PATCH 4/6] Disable tests on darwin --- src/restic/node_unix_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/restic/node_unix_test.go b/src/restic/node_unix_test.go index a581b3466..8ed3ea0bb 100644 --- a/src/restic/node_unix_test.go +++ b/src/restic/node_unix_test.go @@ -1,4 +1,4 @@ -// +build !windows +// +build !windows,!darwin package restic_test From 10a395ca338b834953e6f490f56cedb62dc45fbb Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Thu, 6 Apr 2017 20:36:09 +0200 Subject: [PATCH 5/6] Make tests runnable on os x/darwin --- src/restic/node_unix_test.go | 55 +++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/src/restic/node_unix_test.go b/src/restic/node_unix_test.go index 8ed3ea0bb..0b8d20434 100644 --- a/src/restic/node_unix_test.go +++ b/src/restic/node_unix_test.go @@ -1,10 +1,9 @@ -// +build !windows,!darwin +// +build !windows -package restic_test +package restic import ( "os" - "restic" "syscall" "testing" "time" @@ -23,16 +22,16 @@ func stat(t testing.TB, filename string) (fi os.FileInfo, ok bool) { return fi, true } -func checkFile(t testing.TB, stat *syscall.Stat_t, node *restic.Node) { - if uint32(node.Mode.Perm()) != stat.Mode&0777 { +func checkFile(t testing.TB, stat *syscall.Stat_t, node *Node) { + if uint32(node.Mode.Perm()) != uint32(stat.Mode&0777) { t.Errorf("Mode does not match, want %v, got %v", stat.Mode&0777, node.Mode) } - if node.Inode != stat.Ino { + if node.Inode != uint64(stat.Ino) { t.Errorf("Inode does not match, want %v, got %v", stat.Ino, node.Inode) } - if node.DeviceID != stat.Dev { + if node.DeviceID != uint64(stat.Dev) { t.Errorf("Dev does not match, want %v, got %v", stat.Dev, node.DeviceID) } @@ -40,22 +39,10 @@ func checkFile(t testing.TB, stat *syscall.Stat_t, node *restic.Node) { t.Errorf("Size does not match, want %v, got %v", stat.Size, node.Size) } - if node.Links != stat.Nlink { + if node.Links != uint64(stat.Nlink) { t.Errorf("Links does not match, want %v, got %v", stat.Nlink, node.Links) } - if node.ModTime != time.Unix(stat.Mtim.Unix()) { - t.Errorf("ModTime does not match, want %v, got %v", time.Unix(stat.Mtim.Unix()), node.ModTime) - } - - if node.ChangeTime != time.Unix(stat.Ctim.Unix()) { - t.Errorf("ChangeTime does not match, want %v, got %v", time.Unix(stat.Ctim.Unix()), node.ChangeTime) - } - - if node.AccessTime != time.Unix(stat.Atim.Unix()) { - t.Errorf("AccessTime does not match, want %v, got %v", time.Unix(stat.Atim.Unix()), node.AccessTime) - } - if node.UID != stat.Uid { t.Errorf("UID does not match, want %v, got %v", stat.Uid, node.UID) } @@ -63,10 +50,32 @@ func checkFile(t testing.TB, stat *syscall.Stat_t, node *restic.Node) { if node.GID != stat.Gid { t.Errorf("UID does not match, want %v, got %v", stat.Gid, node.GID) } + + // use the os dependent function to compare the timestamps + s, ok := toStatT(stat) + if !ok { + return + } + + mtime := s.mtim() + if node.ModTime != time.Unix(mtime.Unix()) { + t.Errorf("ModTime does not match, want %v, got %v", time.Unix(mtime.Unix()), node.ModTime) + } + + ctime := s.ctim() + if node.ChangeTime != time.Unix(ctime.Unix()) { + t.Errorf("ChangeTime does not match, want %v, got %v", time.Unix(ctime.Unix()), node.ChangeTime) + } + + atime := s.atim() + if node.AccessTime != time.Unix(atime.Unix()) { + t.Errorf("AccessTime does not match, want %v, got %v", time.Unix(atime.Unix()), node.AccessTime) + } + } -func checkDevice(t testing.TB, stat *syscall.Stat_t, node *restic.Node) { - if node.Device != stat.Rdev { +func checkDevice(t testing.TB, stat *syscall.Stat_t, node *Node) { + if node.Device != uint64(stat.Rdev) { t.Errorf("Rdev does not match, want %v, got %v", stat.Rdev, node.Device) } } @@ -100,7 +109,7 @@ func TestNodeFromFileInfo(t *testing.T) { return } - node, err := restic.NodeFromFileInfo(test.filename, fi) + node, err := NodeFromFileInfo(test.filename, fi) if err != nil { t.Fatal(err) } From db7e23b4234f5eb5a814c5a578a798fe1244ec46 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Fri, 7 Apr 2017 20:37:20 +0200 Subject: [PATCH 6/6] Skip /dev/null on darwin --- src/restic/node_unix_test.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/restic/node_unix_test.go b/src/restic/node_unix_test.go index 0b8d20434..a9f0c5df1 100644 --- a/src/restic/node_unix_test.go +++ b/src/restic/node_unix_test.go @@ -4,6 +4,7 @@ package restic import ( "os" + "runtime" "syscall" "testing" "time" @@ -81,15 +82,21 @@ func checkDevice(t testing.TB, stat *syscall.Stat_t, node *Node) { } func TestNodeFromFileInfo(t *testing.T) { - var tests = []struct { + type Test struct { filename string canSkip bool - }{ + } + var tests = []Test{ {"node_test.go", false}, - {"/dev/null", true}, {"/dev/sda", true}, } + // on darwin, users are not permitted to list the extended attributes of + // /dev/null, therefore skip it. + if runtime.GOOS != "darwin" { + tests = append(tests, Test{"/dev/null", true}) + } + for _, test := range tests { t.Run("", func(t *testing.T) { fi, found := stat(t, test.filename)