diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dfbd7a7ed..b6d36face 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ permissions: contents: read env: - latest_go: "1.21.x" + latest_go: "1.22.x" GO111MODULE: on jobs: @@ -23,18 +23,18 @@ jobs: # list of jobs to run: include: - job_name: Windows - go: 1.21.x + go: 1.22.x os: windows-latest test_smb: true - job_name: macOS - go: 1.21.x + go: 1.22.x os: macOS-latest test_fuse: false test_smb: true - job_name: Linux - go: 1.21.x + go: 1.22.x os: ubuntu-latest test_cloud_backends: true test_fuse: true @@ -42,12 +42,17 @@ jobs: check_changelog: true - job_name: Linux (race) - go: 1.21.x + go: 1.22.x os: ubuntu-latest test_fuse: true test_smb: true test_opts: "-race" + - job_name: Linux + go: 1.21.x + os: ubuntu-latest + test_fuse: true + - job_name: Linux go: 1.20.x os: ubuntu-latest @@ -373,7 +378,7 @@ jobs: uses: golangci/golangci-lint-action@v3 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.55.2 + version: v1.56.1 args: --verbose --timeout 5m # only run golangci-lint for pull requests, otherwise ALL hints get diff --git a/.golangci.yml b/.golangci.yml index 98b5f9e03..7dc6a8e7f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -35,6 +35,9 @@ linters: # parse and typecheck code - typecheck + # ensure that http response bodies are closed + - bodyclose + issues: # don't use the default exclude rules, this hides (among others) ignored # errors from Close() calls @@ -51,3 +54,8 @@ issues: # staticcheck: there's no easy way to replace these packages - "SA1019: \"golang.org/x/crypto/poly1305\" is deprecated" - "SA1019: \"golang.org/x/crypto/openpgp\" is deprecated" + + exclude-rules: + # revive: ignore unused parameters in tests + - path: (_test\.go|testing\.go|backend/.*/tests\.go) + text: "unused-parameter:" \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index c999b090d..b8969a443 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Table of Contents +* [Changelog for 0.16.4](#changelog-for-restic-0164-2024-02-04) +* [Changelog for 0.16.3](#changelog-for-restic-0163-2024-01-14) * [Changelog for 0.16.2](#changelog-for-restic-0162-2023-10-29) * [Changelog for 0.16.1](#changelog-for-restic-0161-2023-10-24) * [Changelog for 0.16.0](#changelog-for-restic-0160-2023-07-31) @@ -31,6 +33,135 @@ * [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29) +# Changelog for restic 0.16.4 (2024-02-04) +The following sections list the changes in restic 0.16.4 relevant to +restic users. The changes are ordered by importance. + +## Summary + + * Fix #4677: Downgrade zstd library to fix rare data corruption at max. compression + * Enh #4529: Add extra verification of data integrity before upload + +## Details + + * Bugfix #4677: Downgrade zstd library to fix rare data corruption at max. compression + + In restic 0.16.3, backups where the compression level was set to `max` (using + `--compression max`) could in rare and very specific circumstances result in + data corruption due to a bug in the library used for compressing data. Restic + 0.16.1 and 0.16.2 were not affected. + + Restic now uses the previous version of the library used to compress data, the + same version used by restic 0.16.2. Please note that the `auto` compression + level (which restic uses by default) was never affected, and even if you used + `max` compression, chances of being affected by this issue are small. + + To check a repository for any corruption, run `restic check --read-data`. This + will download and verify the whole repository and can be used at any time to + completely verify the integrity of a repository. If the `check` command detects + anomalies, follow the suggested steps. + + https://github.com/restic/restic/issues/4677 + https://github.com/restic/restic/pull/4679 + + * Enhancement #4529: Add extra verification of data integrity before upload + + Hardware issues, or a bug in restic or its dependencies, could previously cause + corruption in the files restic created and stored in the repository. Detecting + such corruption previously required explicitly running the `check --read-data` + or `check --read-data-subset` commands. + + To further ensure data integrity, even in the case of hardware issues or + software bugs, restic now performs additional verification of the files about to + be uploaded to the repository. + + These extra checks will increase CPU usage during backups. They can therefore, + if absolutely necessary, be disabled using the `--no-extra-verify` global + option. Please note that this should be combined with more active checking using + the previously mentioned check commands. + + https://github.com/restic/restic/issues/4529 + https://github.com/restic/restic/pull/4681 + + +# Changelog for restic 0.16.3 (2024-01-14) +The following sections list the changes in restic 0.16.3 relevant to +restic users. The changes are ordered by importance. + +## Summary + + * Fix #4560: Improve errors for irregular files on Windows + * Fix #4574: Support backup of deduplicated files on Windows again + * Fix #4612: Improve error handling for `rclone` backend + * Fix #4624: Correct `restore` progress information if an error occurs + * Fix #4626: Improve reliability of restoring large files + +## Details + + * Bugfix #4560: Improve errors for irregular files on Windows + + Since Go 1.21, most filesystem reparse points on Windows are considered to be + irregular files. This caused restic to show an `error: invalid node type ""` + error message for those files. + + This error message has now been improved and includes the relevant file path: + `error: nodeFromFileInfo path/to/file: unsupported file type "irregular"`. As + irregular files are not required to behave like regular files, it is not + possible to provide a generic way to back up those files. + + https://github.com/restic/restic/issues/4560 + https://github.com/restic/restic/pull/4620 + https://forum.restic.net/t/windows-backup-error-invalid-node-type/6875 + + * Bugfix #4574: Support backup of deduplicated files on Windows again + + With the official release builds of restic 0.16.1 and 0.16.2, it was not + possible to back up files that were deduplicated by the corresponding Windows + Server feature. This also applied to restic versions built using Go + 1.21.0-1.21.4. + + The Go version used to build restic has now been updated to fix this. + + https://github.com/restic/restic/issues/4574 + https://github.com/restic/restic/pull/4621 + + * Bugfix #4612: Improve error handling for `rclone` backend + + Since restic 0.16.0, if rclone encountered an error while listing files, this + could in rare circumstances cause restic to assume that there are no files. + Although unlikely, this situation could result in data loss if it were to happen + right when the `prune` command is listing existing snapshots. + + Error handling has now been improved to detect and work around this case. + + https://github.com/restic/restic/issues/4612 + https://github.com/restic/restic/pull/4618 + + * Bugfix #4624: Correct `restore` progress information if an error occurs + + If an error occurred while restoring a snapshot, this could cause the `restore` + progress bar to show incorrect information. In addition, if a data file could + not be loaded completely, then errors would also be reported for some already + restored files. + + Error reporting of the `restore` command has now been made more accurate. + + https://github.com/restic/restic/pull/4624 + https://forum.restic.net/t/errors-restoring-with-restic-on-windows-server-s3/6943 + + * Bugfix #4626: Improve reliability of restoring large files + + In some cases restic failed to restore large files that frequently contain the + same file chunk. In combination with certain backends, this could result in + network connection timeouts that caused incomplete restores. + + Restic now includes special handling for such file chunks to ensure reliable + restores. + + https://github.com/restic/restic/pull/4626 + https://forum.restic.net/t/errors-restoring-with-restic-on-windows-server-s3/6943 + + # Changelog for restic 0.16.2 (2023-10-29) The following sections list the changes in restic 0.16.2 relevant to restic users. The changes are ordered by importance. @@ -44,16 +175,18 @@ restic users. The changes are ordered by importance. * Bugfix #4540: Restore ARMv5 support for ARM binaries - The official release binaries for restic 0.16.1 were accidentally built to require ARMv7. The - build process is now updated to restore support for ARMv5. + The official release binaries for restic 0.16.1 were accidentally built to + require ARMv7. The build process is now updated to restore support for ARMv5. - Please note that restic 0.17.0 will drop support for ARMv5 and require at least ARMv6. + Please note that restic 0.17.0 will drop support for ARMv5 and require at least + ARMv6. https://github.com/restic/restic/issues/4540 * Bugfix #4545: Repair documentation build on Read the Docs - For restic 0.16.1, no documentation was available at https://restic.readthedocs.io/ . + For restic 0.16.1, no documentation was available at + https://restic.readthedocs.io/ . The documentation build process is now updated to work again. @@ -80,65 +213,67 @@ restic users. The changes are ordered by importance. * Bugfix #4513: Make `key list` command honor `--no-lock` - The `key list` command now supports the `--no-lock` options. This allows determining which - keys a repo can be accessed by without the need for having write access (e.g., read-only sftp - access, filesystem snapshot). + The `key list` command now supports the `--no-lock` options. This allows + determining which keys a repo can be accessed by without the need for having + write access (e.g., read-only sftp access, filesystem snapshot). https://github.com/restic/restic/issues/4513 https://github.com/restic/restic/pull/4514 * Bugfix #4516: Do not try to load password on command line autocomplete - The command line autocompletion previously tried to load the repository password. This could - cause the autocompletion not to work. Now, this step gets skipped. + The command line autocompletion previously tried to load the repository + password. This could cause the autocompletion not to work. Now, this step gets + skipped. https://github.com/restic/restic/issues/4516 https://github.com/restic/restic/pull/4526 * Bugfix #4523: Update zstd library to fix possible data corruption at max. compression - In restic 0.16.0, backups where the compression level was set to `max` (using `--compression - max`) could in rare and very specific circumstances result in data corruption due to a bug in the - library used for compressing data. + In restic 0.16.0, backups where the compression level was set to `max` (using + `--compression max`) could in rare and very specific circumstances result in + data corruption due to a bug in the library used for compressing data. - Restic now uses the latest version of the library used to compress data, which includes a fix for - this issue. Please note that the `auto` compression level (which restic uses by default) was - never affected, and even if you used `max` compression, chances of being affected by this issue - were very small. + Restic now uses the latest version of the library used to compress data, which + includes a fix for this issue. Please note that the `auto` compression level + (which restic uses by default) was never affected, and even if you used `max` + compression, chances of being affected by this issue were very small. - To check a repository for any corruption, run `restic check --read-data`. This will download - and verify the whole repository and can be used at any time to completely verify the integrity of - a repository. If the `check` command detects anomalies, follow the suggested steps. + To check a repository for any corruption, run `restic check --read-data`. This + will download and verify the whole repository and can be used at any time to + completely verify the integrity of a repository. If the `check` command detects + anomalies, follow the suggested steps. - To simplify any needed repository repair and minimize data loss, there is also a new and - experimental `repair packs` command that salvages all valid data from the affected pack files - (see `restic help repair packs` for more information). + To simplify any needed repository repair and minimize data loss, there is also a + new and experimental `repair packs` command that salvages all valid data from + the affected pack files (see `restic help repair packs` for more information). https://github.com/restic/restic/issues/4523 https://github.com/restic/restic/pull/4530 * Change #4532: Update dependencies and require Go 1.19 or newer - We have updated all dependencies. Since some libraries require newer Go standard library - features, support for Go 1.18 has been dropped, which means that restic now requires at least Go - 1.19 to build. + We have updated all dependencies. Since some libraries require newer Go standard + library features, support for Go 1.18 has been dropped, which means that restic + now requires at least Go 1.19 to build. https://github.com/restic/restic/pull/4532 https://github.com/restic/restic/pull/4533 * Enhancement #229: Show progress bar while loading the index - Restic did not provide any feedback while loading index files. Now, there is a progress bar that - shows the index loading progress. + Restic did not provide any feedback while loading index files. Now, there is a + progress bar that shows the index loading progress. https://github.com/restic/restic/issues/229 https://github.com/restic/restic/pull/4419 * Enhancement #4128: Automatically set `GOMAXPROCS` in resource-constrained containers - When running restic in a Linux container with CPU-usage limits, restic now automatically - adjusts `GOMAXPROCS`. This helps to reduce the memory consumption on hosts with many CPU - cores. + When running restic in a Linux container with CPU-usage limits, restic now + automatically adjusts `GOMAXPROCS`. This helps to reduce the memory consumption + on hosts with many CPU cores. https://github.com/restic/restic/issues/4128 https://github.com/restic/restic/pull/4485 @@ -146,32 +281,33 @@ restic users. The changes are ordered by importance. * Enhancement #4480: Allow setting REST password and username via environment variables - Previously, it was only possible to specify the REST-server username and password in the - repository URL, or by using the `--repository-file` option. This meant it was not possible to - use authentication in contexts where the repository URL is stored in publicly accessible way. + Previously, it was only possible to specify the REST-server username and + password in the repository URL, or by using the `--repository-file` option. This + meant it was not possible to use authentication in contexts where the repository + URL is stored in publicly accessible way. - Restic now allows setting the username and password using the `RESTIC_REST_USERNAME` and - `RESTIC_REST_PASSWORD` variables. + Restic now allows setting the username and password using the + `RESTIC_REST_USERNAME` and `RESTIC_REST_PASSWORD` variables. https://github.com/restic/restic/pull/4480 * Enhancement #4511: Include inode numbers in JSON output for `find` and `ls` commands - Restic used to omit the inode numbers in the JSON messages emitted for nodes by the `ls` command - as well as for matches by the `find` command. It now includes those values whenever they are - available. + Restic used to omit the inode numbers in the JSON messages emitted for nodes by + the `ls` command as well as for matches by the `find` command. It now includes + those values whenever they are available. https://github.com/restic/restic/pull/4511 * Enhancement #4519: Add config option to set SFTP command arguments - When using the `sftp` backend, scenarios where a custom identity file was needed for the SSH - connection, required the full command to be specified: `-o sftp.command='ssh - user@host:port -i /ssh/my_private_key -s sftp'` + When using the `sftp` backend, scenarios where a custom identity file was needed + for the SSH connection, required the full command to be specified: `-o + sftp.command='ssh user@host:port -i /ssh/my_private_key -s sftp'` - Now, the `-o sftp.args=...` option can be passed to restic to specify custom arguments for the - SSH command executed by the SFTP backend. This simplifies the above example to `-o - sftp.args='-i /ssh/my_private_key'`. + Now, the `-o sftp.args=...` option can be passed to restic to specify custom + arguments for the SSH command executed by the SFTP backend. This simplifies the + above example to `-o sftp.args='-i /ssh/my_private_key'`. https://github.com/restic/restic/issues/4241 https://github.com/restic/restic/pull/4519 @@ -217,31 +353,32 @@ restic users. The changes are ordered by importance. * Bugfix #2565: Support "unlimited" in `forget --keep-*` options - Restic would previously forget snapshots that should have been kept when a negative value was - passed to the `--keep-*` options. Negative values are now forbidden. To keep all snapshots, - the special value `unlimited` is now supported. For example, `--keep-monthly unlimited` - will keep all monthly snapshots. + Restic would previously forget snapshots that should have been kept when a + negative value was passed to the `--keep-*` options. Negative values are now + forbidden. To keep all snapshots, the special value `unlimited` is now + supported. For example, `--keep-monthly unlimited` will keep all monthly + snapshots. https://github.com/restic/restic/issues/2565 https://github.com/restic/restic/pull/4234 * Bugfix #3311: Support non-UTF8 paths as symlink target - Earlier restic versions did not correctly `backup` and `restore` symlinks that contain a - non-UTF8 target. Note that this only affected systems that still use a non-Unicode encoding - for filesystem paths. + Earlier restic versions did not correctly `backup` and `restore` symlinks that + contain a non-UTF8 target. Note that this only affected systems that still use a + non-Unicode encoding for filesystem paths. - The repository format is now extended to add support for such symlinks. Please note that - snapshots must have been created with at least restic version 0.16.0 for `restore` to - correctly handle non-UTF8 symlink targets when restoring them. + The repository format is now extended to add support for such symlinks. Please + note that snapshots must have been created with at least restic version 0.16.0 + for `restore` to correctly handle non-UTF8 symlink targets when restoring them. https://github.com/restic/restic/issues/3311 https://github.com/restic/restic/pull/3802 * Bugfix #4199: Avoid lock refresh issues on slow network connections - On network connections with a low upload speed, backups and other operations could fail with - the error message `Fatal: failed to refresh lock in time`. + On network connections with a low upload speed, backups and other operations + could fail with the error message `Fatal: failed to refresh lock in time`. This has now been fixed by reworking the lock refresh handling. @@ -250,21 +387,21 @@ restic users. The changes are ordered by importance. * Bugfix #4274: Improve lock refresh handling after standby - If the restic process was stopped or the host running restic entered standby during a long - running operation such as a backup, this previously resulted in the operation failing with - `Fatal: failed to refresh lock in time`. + If the restic process was stopped or the host running restic entered standby + during a long running operation such as a backup, this previously resulted in + the operation failing with `Fatal: failed to refresh lock in time`. - This has now been fixed such that restic first checks whether it is safe to continue the current - operation and only throws an error if not. + This has now been fixed such that restic first checks whether it is safe to + continue the current operation and only throws an error if not. https://github.com/restic/restic/issues/4274 https://github.com/restic/restic/pull/4374 * Bugfix #4319: Correctly clean up status bar output of the `backup` command - Due to a regression in restic 0.15.2, the status bar of the `backup` command could leave some - output behind. This happened if filenames were printed that are wider than the current - terminal width. This has now been fixed. + Due to a regression in restic 0.15.2, the status bar of the `backup` command + could leave some output behind. This happened if filenames were printed that are + wider than the current terminal width. This has now been fixed. https://github.com/restic/restic/issues/4319 https://github.com/restic/restic/pull/4318 @@ -275,25 +412,26 @@ restic users. The changes are ordered by importance. * Bugfix #4400: Ignore missing folders in `rest` backend - If a repository accessed via the REST backend was missing folders, then restic would fail with - an error while trying to list the data in the repository. This has been now fixed. + If a repository accessed via the REST backend was missing folders, then restic + would fail with an error while trying to list the data in the repository. This + has been now fixed. https://github.com/restic/rest-server/issues/235 https://github.com/restic/restic/pull/4400 * Change #4176: Fix JSON message type of `scan_finished` for the `backup` command - Restic incorrectly set the `message_type` of the `scan_finished` message to `status` - instead of `verbose_status`. This has now been corrected so that the messages report the - correct type. + Restic incorrectly set the `message_type` of the `scan_finished` message to + `status` instead of `verbose_status`. This has now been corrected so that the + messages report the correct type. https://github.com/restic/restic/pull/4176 * Change #4201: Require Go 1.20 for Solaris builds - Building restic on Solaris now requires Go 1.20, as the library used to access Azure uses the - mmap syscall, which is only available on Solaris starting from Go 1.20. All other platforms - however continue to build with Go 1.18. + Building restic on Solaris now requires Go 1.20, as the library used to access + Azure uses the mmap syscall, which is only available on Solaris starting from Go + 1.20. All other platforms however continue to build with Go 1.18. https://github.com/restic/restic/pull/4201 @@ -314,8 +452,8 @@ restic users. The changes are ordered by importance. * Enhancement #719: Add `--retry-lock` option - This option allows specifying a duration for which restic will wait if the repository is - already locked. + This option allows specifying a duration for which restic will wait if the + repository is already locked. https://github.com/restic/restic/issues/719 https://github.com/restic/restic/pull/2214 @@ -323,24 +461,25 @@ restic users. The changes are ordered by importance. * Enhancement #1495: Sort snapshots by timestamp in `restic find` - The `find` command used to print snapshots in an arbitrary order. Restic now prints snapshots - sorted by timestamp. + The `find` command used to print snapshots in an arbitrary order. Restic now + prints snapshots sorted by timestamp. https://github.com/restic/restic/issues/1495 https://github.com/restic/restic/pull/4409 * Enhancement #1759: Add `repair index` and `repair snapshots` commands - The `rebuild-index` command has been renamed to `repair index`. The old name will still work, - but is deprecated. + The `rebuild-index` command has been renamed to `repair index`. The old name + will still work, but is deprecated. - When a snapshot was damaged, the only option up to now was to completely forget the snapshot, - even if only some unimportant files in it were damaged and other files were still fine. + When a snapshot was damaged, the only option up to now was to completely forget + the snapshot, even if only some unimportant files in it were damaged and other + files were still fine. - Restic now has a `repair snapshots` command, which can salvage any non-damaged files and parts - of files in the snapshots by removing damaged directories and missing file contents. Please - note that the damaged data may still be lost and see the "Troubleshooting" section in the - documentation for more details. + Restic now has a `repair snapshots` command, which can salvage any non-damaged + files and parts of files in the snapshots by removing damaged directories and + missing file contents. Please note that the damaged data may still be lost and + see the "Troubleshooting" section in the documentation for more details. https://github.com/restic/restic/issues/1759 https://github.com/restic/restic/issues/1714 @@ -352,19 +491,20 @@ restic users. The changes are ordered by importance. * Enhancement #1926: Allow certificate paths to be passed through environment variables - Restic will now read paths to certificates from the environment variables `RESTIC_CACERT` or - `RESTIC_TLS_CLIENT_CERT` if `--cacert` or `--tls-client-cert` are not specified. + Restic will now read paths to certificates from the environment variables + `RESTIC_CACERT` or `RESTIC_TLS_CLIENT_CERT` if `--cacert` or `--tls-client-cert` + are not specified. https://github.com/restic/restic/issues/1926 https://github.com/restic/restic/pull/4384 * Enhancement #2359: Provide multi-platform Docker images - The official Docker images are now built for the architectures linux/386, linux/amd64, - linux/arm and linux/arm64. + The official Docker images are now built for the architectures linux/386, + linux/amd64, linux/arm and linux/arm64. - As an alternative to the Docker Hub, the Docker images are also available on ghcr.io, the GitHub - Container Registry. + As an alternative to the Docker Hub, the Docker images are also available on + ghcr.io, the GitHub Container Registry. https://github.com/restic/restic/issues/2359 https://github.com/restic/restic/issues/4269 @@ -374,25 +514,26 @@ restic users. The changes are ordered by importance. The `azure` backend previously only supported storages using the global domain `core.windows.net`. This meant that backups to other domains such as Azure China - (`core.chinacloudapi.cn`) or Azure Germany (`core.cloudapi.de`) were not supported. - Restic now allows overriding the global domain using the environment variable - `AZURE_ENDPOINT_SUFFIX`. + (`core.chinacloudapi.cn`) or Azure Germany (`core.cloudapi.de`) were not + supported. Restic now allows overriding the global domain using the environment + variable `AZURE_ENDPOINT_SUFFIX`. https://github.com/restic/restic/issues/2468 https://github.com/restic/restic/pull/4387 * Enhancement #2679: Reduce file fragmentation for local backend - Before this change, local backend files could become fragmented. Now restic will try to - preallocate space for pack files to avoid their fragmentation. + Before this change, local backend files could become fragmented. Now restic will + try to preallocate space for pack files to avoid their fragmentation. https://github.com/restic/restic/issues/2679 https://github.com/restic/restic/pull/3261 * Enhancement #3328: Reduce memory usage by up to 25% - The in-memory index has been optimized to be more garbage collection friendly. Restic now - defaults to `GOGC=50` to run the Go garbage collector more frequently. + The in-memory index has been optimized to be more garbage collection friendly. + Restic now defaults to `GOGC=50` to run the Go garbage collector more + frequently. https://github.com/restic/restic/issues/3328 https://github.com/restic/restic/pull/4352 @@ -400,21 +541,21 @@ restic users. The changes are ordered by importance. * Enhancement #3397: Improve accuracy of ETA displayed during backup - Restic's `backup` command displayed an ETA that did not adapt when the rate of progress made - during the backup changed during the course of the backup. + Restic's `backup` command displayed an ETA that did not adapt when the rate of + progress made during the backup changed during the course of the backup. - Restic now uses recent progress when computing the ETA. It is important to realize that the - estimate may still be wrong, because restic cannot predict the future, but the hope is that the - ETA will be more accurate in most cases. + Restic now uses recent progress when computing the ETA. It is important to + realize that the estimate may still be wrong, because restic cannot predict the + future, but the hope is that the ETA will be more accurate in most cases. https://github.com/restic/restic/issues/3397 https://github.com/restic/restic/pull/3563 * Enhancement #3624: Keep oldest snapshot when there are not enough snapshots - The `forget` command now additionally preserves the oldest snapshot if fewer snapshots than - allowed by the `--keep-*` parameters would otherwise be kept. This maximizes the amount of - history kept within the specified limits. + The `forget` command now additionally preserves the oldest snapshot if fewer + snapshots than allowed by the `--keep-*` parameters would otherwise be kept. + This maximizes the amount of history kept within the specified limits. https://github.com/restic/restic/issues/3624 https://github.com/restic/restic/pull/4366 @@ -422,99 +563,106 @@ restic users. The changes are ordered by importance. * Enhancement #3698: Add support for Managed / Workload Identity to `azure` backend - Restic now additionally supports authenticating to Azure using Workload Identity or Managed - Identity credentials, which are automatically injected in several environments such as a - managed Kubernetes cluster. + Restic now additionally supports authenticating to Azure using Workload Identity + or Managed Identity credentials, which are automatically injected in several + environments such as a managed Kubernetes cluster. https://github.com/restic/restic/issues/3698 https://github.com/restic/restic/pull/4029 * Enhancement #3871: Support `:` syntax to select subfolders - Commands like `diff` or `restore` always worked with the full snapshot. This did not allow - comparing only a specific subfolder or only restoring that folder (`restore --include - subfolder` filters the restored files, but still creates the directories included in - `subfolder`). + Commands like `diff` or `restore` always worked with the full snapshot. This did + not allow comparing only a specific subfolder or only restoring that folder + (`restore --include subfolder` filters the restored files, but still creates the + directories included in `subfolder`). - The commands `diff`, `dump`, `ls` and `restore` now support the `:` - syntax, where `snapshot` is the ID of a snapshot (or the string `latest`) and `subfolder` is a - path within the snapshot. The commands will then only work with the specified path of the - snapshot. The `subfolder` must be a path to a folder as returned by `ls`. Two examples: + The commands `diff`, `dump`, `ls` and `restore` now support the + `:` syntax, where `snapshot` is the ID of a snapshot (or + the string `latest`) and `subfolder` is a path within the snapshot. The commands + will then only work with the specified path of the snapshot. The `subfolder` + must be a path to a folder as returned by `ls`. Two examples: `restic restore -t target latest:/some/path` `restic diff 12345678:/some/path 90abcef:/some/path` - For debugging purposes, the `cat` command now supports `cat tree :` to - return the directory metadata for the given subfolder. + For debugging purposes, the `cat` command now supports `cat tree + :` to return the directory metadata for the given + subfolder. https://github.com/restic/restic/issues/3871 https://github.com/restic/restic/pull/4334 * Enhancement #3941: Support `--group-by` for backup parent selection - Previously, the `backup` command by default selected the parent snapshot based on the - hostname and the backup targets. When the backup path list changed, the `backup` command was - unable to determine a suitable parent snapshot and had to read all files again. + Previously, the `backup` command by default selected the parent snapshot based + on the hostname and the backup targets. When the backup path list changed, the + `backup` command was unable to determine a suitable parent snapshot and had to + read all files again. - The new `--group-by` option for the `backup` command allows filtering snapshots for the - parent selection by `host`, `paths` and `tags`. It defaults to `host,paths` which selects the - latest snapshot with hostname and paths matching those of the backup run. This matches the - behavior of prior restic versions. + The new `--group-by` option for the `backup` command allows filtering snapshots + for the parent selection by `host`, `paths` and `tags`. It defaults to + `host,paths` which selects the latest snapshot with hostname and paths matching + those of the backup run. This matches the behavior of prior restic versions. - The new `--group-by` option should be set to the same value as passed to `forget --group-by`. + The new `--group-by` option should be set to the same value as passed to `forget + --group-by`. https://github.com/restic/restic/issues/3941 https://github.com/restic/restic/pull/4081 * Enhancement #4130: Cancel current command if cache becomes unusable - If the cache directory was removed or ran out of space while restic was running, this would - previously cause further caching attempts to fail and thereby drastically slow down the - command execution. Now, the currently running command is instead canceled. + If the cache directory was removed or ran out of space while restic was running, + this would previously cause further caching attempts to fail and thereby + drastically slow down the command execution. Now, the currently running command + is instead canceled. https://github.com/restic/restic/issues/4130 https://github.com/restic/restic/pull/4166 * Enhancement #4159: Add `--human-readable` option to `ls` and `find` commands - Previously, when using the `-l` option with the `ls` and `find` commands, the displayed size - was always in bytes, without an option for a more human readable format such as MiB or GiB. + Previously, when using the `-l` option with the `ls` and `find` commands, the + displayed size was always in bytes, without an option for a more human readable + format such as MiB or GiB. - The new `--human-readable` option will convert longer size values into more human friendly - values with an appropriate suffix depending on the output size. For example, a size of - `14680064` will be shown as `14.000 MiB`. + The new `--human-readable` option will convert longer size values into more + human friendly values with an appropriate suffix depending on the output size. + For example, a size of `14680064` will be shown as `14.000 MiB`. https://github.com/restic/restic/issues/4159 https://github.com/restic/restic/pull/4351 * Enhancement #4188: Include restic version in snapshot metadata - The restic version used to backup a snapshot is now included in its metadata and shown when - inspecting a snapshot using `restic cat snapshot ` or `restic snapshots - --json`. + The restic version used to backup a snapshot is now included in its metadata and + shown when inspecting a snapshot using `restic cat snapshot ` or + `restic snapshots --json`. https://github.com/restic/restic/issues/4188 https://github.com/restic/restic/pull/4378 * Enhancement #4220: Add `jq` binary to Docker image - The Docker image now contains `jq`, which can be useful to process JSON data output by restic. + The Docker image now contains `jq`, which can be useful to process JSON data + output by restic. https://github.com/restic/restic/pull/4220 * Enhancement #4226: Allow specifying region of new buckets in the `gs` backend - Previously, buckets used by the Google Cloud Storage backend would always get created in the - "us" region. It is now possible to specify the region where a bucket should be created by using - the `-o gs.region=us` option. + Previously, buckets used by the Google Cloud Storage backend would always get + created in the "us" region. It is now possible to specify the region where a + bucket should be created by using the `-o gs.region=us` option. https://github.com/restic/restic/pull/4226 * Enhancement #4375: Add support for extended attributes on symlinks - Restic now supports extended attributes on symlinks when backing up, restoring, or - FUSE-mounting snapshots. This includes, for example, the `security.selinux` xattr on Linux - distributions that use SELinux. + Restic now supports extended attributes on symlinks when backing up, restoring, + or FUSE-mounting snapshots. This includes, for example, the `security.selinux` + xattr on Linux distributions that use SELinux. https://github.com/restic/restic/issues/4375 https://github.com/restic/restic/pull/4379 @@ -543,12 +691,12 @@ restic users. The changes are ordered by importance. * Bugfix #2260: Sanitize filenames printed by `backup` during processing - The `backup` command would previously not sanitize the filenames it printed during - processing, potentially causing newlines or terminal control characters to mangle the - status output or even change the state of a terminal. + The `backup` command would previously not sanitize the filenames it printed + during processing, potentially causing newlines or terminal control characters + to mangle the status output or even change the state of a terminal. - Filenames are now checked and quoted if they contain non-printable or non-Unicode - characters. + Filenames are now checked and quoted if they contain non-printable or + non-Unicode characters. https://github.com/restic/restic/issues/2260 https://github.com/restic/restic/issues/4191 @@ -557,44 +705,47 @@ restic users. The changes are ordered by importance. * Bugfix #4211: Make `dump` interpret `--host` and `--path` correctly A regression in restic 0.15.0 caused `dump` to confuse its `--host=` and - `--path=` options: it looked for snapshots with paths called `` from hosts - called ``. It now treats the options as intended. + `--path=` options: it looked for snapshots with paths called `` from + hosts called ``. It now treats the options as intended. https://github.com/restic/restic/issues/4211 https://github.com/restic/restic/pull/4212 * Bugfix #4239: Correct number of blocks reported in mount point - Restic mount points reported an incorrect number of 512-byte (POSIX standard) blocks for - files and links due to a rounding bug. In particular, empty files were reported as taking one - block instead of zero. + Restic mount points reported an incorrect number of 512-byte (POSIX standard) + blocks for files and links due to a rounding bug. In particular, empty files + were reported as taking one block instead of zero. - The rounding is now fixed: the number of blocks reported is the file size (or link target size) - divided by 512 and rounded up to a whole number. + The rounding is now fixed: the number of blocks reported is the file size (or + link target size) divided by 512 and rounded up to a whole number. https://github.com/restic/restic/issues/4239 https://github.com/restic/restic/pull/4240 * Bugfix #4253: Minimize risk of spurious filesystem loops with `mount` - When a backup contains a directory that has the same name as its parent, say `a/b/b`, and the GNU - `find` command was run on this backup in a restic mount, `find` would refuse to traverse the - lowest `b` directory, instead printing `File system loop detected`. This was due to the way the - restic mount command generates inode numbers for directories in the mount point. + When a backup contains a directory that has the same name as its parent, say + `a/b/b`, and the GNU `find` command was run on this backup in a restic mount, + `find` would refuse to traverse the lowest `b` directory, instead printing `File + system loop detected`. This was due to the way the restic mount command + generates inode numbers for directories in the mount point. - The rule for generating these inode numbers was changed in 0.15.0. It has now been changed again - to avoid this issue. A perfect rule does not exist, but the probability of this behavior - occurring is now extremely small. + The rule for generating these inode numbers was changed in 0.15.0. It has now + been changed again to avoid this issue. A perfect rule does not exist, but the + probability of this behavior occurring is now extremely small. - When it does occur, the mount point is not broken, and scripts that traverse the mount point - should work as long as they don't rely on inode numbers for detecting filesystem loops. + When it does occur, the mount point is not broken, and scripts that traverse the + mount point should work as long as they don't rely on inode numbers for + detecting filesystem loops. https://github.com/restic/restic/issues/4253 https://github.com/restic/restic/pull/4255 * Enhancement #4180: Add release binaries for riscv64 architecture on Linux - Builds for the `riscv64` architecture on Linux are now included in the release binaries. + Builds for the `riscv64` architecture on Linux are now included in the release + binaries. https://github.com/restic/restic/pull/4180 @@ -621,8 +772,8 @@ restic users. The changes are ordered by importance. * Bugfix #3750: Remove `b2_download_file_by_name: 404` warning from B2 backend - In some cases the B2 backend could print `b2_download_file_by_name: 404: : b2.b2err` - warnings. These are only debug messages and can be safely ignored. + In some cases the B2 backend could print `b2_download_file_by_name: 404: : + b2.b2err` warnings. These are only debug messages and can be safely ignored. Restic now uses an updated library for accessing B2, which removes the warning. @@ -632,19 +783,19 @@ restic users. The changes are ordered by importance. * Bugfix #4147: Make `prune --quiet` not print progress bar - A regression in restic 0.15.0 caused `prune --quiet` to show a progress bar while deciding how - to process each pack files. This has now been fixed. + A regression in restic 0.15.0 caused `prune --quiet` to show a progress bar + while deciding how to process each pack files. This has now been fixed. https://github.com/restic/restic/issues/4147 https://github.com/restic/restic/pull/4153 * Bugfix #4163: Make `self-update --output` work with new filename on Windows - Since restic 0.14.0 the `self-update` command did not work when a custom output filename was - specified via the `--output` option. This has now been fixed. + Since restic 0.14.0 the `self-update` command did not work when a custom output + filename was specified via the `--output` option. This has now been fixed. - As a workaround, either use an older restic version to run the self-update or create an empty - file with the output filename before updating e.g. using CMD: + As a workaround, either use an older restic version to run the self-update or + create an empty file with the output filename before updating e.g. using CMD: `type nul > new-file.exe` `restic self-update --output new-file.exe` @@ -653,24 +804,27 @@ restic users. The changes are ordered by importance. * Bugfix #4167: Add missing ETA in `backup` progress bar - A regression in restic 0.15.0 caused the ETA to be missing from the progress bar displayed by the - `backup` command. This has now been fixed. + A regression in restic 0.15.0 caused the ETA to be missing from the progress bar + displayed by the `backup` command. This has now been fixed. https://github.com/restic/restic/pull/4167 * Enhancement #4143: Ignore empty lock files - With restic 0.15.0 the checks for stale locks became much stricter than before. In particular, - empty or unreadable locks were no longer silently ignored. This made restic to complain with - `Load(, 0, 0) returned error, retrying after 552.330144ms: - load(): invalid data returned` and fail in the end. + With restic 0.15.0 the checks for stale locks became much stricter than before. + In particular, empty or unreadable locks were no longer silently ignored. This + made restic to complain with `Load(, 0, 0) returned error, + retrying after 552.330144ms: load(): invalid data returned` and + fail in the end. - The error message is now clarified and the implementation changed to ignore empty lock files - which are sometimes created as the result of a failed uploads on some backends. + The error message is now clarified and the implementation changed to ignore + empty lock files which are sometimes created as the result of a failed uploads + on some backends. - Please note that unreadable lock files still have to cleaned up manually. To do so, you can run - `restic unlock --remove-all` which removes all existing lock files. But first make sure that - no other restic process is currently using the repository. + Please note that unreadable lock files still have to cleaned up manually. To do + so, you can run `restic unlock --remove-all` which removes all existing lock + files. But first make sure that no other restic process is currently using the + repository. https://github.com/restic/restic/issues/4143 https://github.com/restic/restic/pull/4152 @@ -726,63 +880,65 @@ restic users. The changes are ordered by importance. * Bugfix #2015: Make `mount` return exit code 0 after receiving Ctrl-C / SIGINT - To stop the `mount` command, a user has to press Ctrl-C or send a SIGINT signal to restic. This - used to cause restic to exit with a non-zero exit code. + To stop the `mount` command, a user has to press Ctrl-C or send a SIGINT signal + to restic. This used to cause restic to exit with a non-zero exit code. - The exit code has now been changed to zero as the above is the expected way to stop the `mount` - command and should therefore be considered successful. + The exit code has now been changed to zero as the above is the expected way to + stop the `mount` command and should therefore be considered successful. https://github.com/restic/restic/issues/2015 https://github.com/restic/restic/pull/3894 * Bugfix #2578: Make `restore` replace existing symlinks - When restoring a symlink, restic used to report an error if the target path already existed. - This has now been fixed such that the potentially existing target path is first removed before - the symlink is restored. + When restoring a symlink, restic used to report an error if the target path + already existed. This has now been fixed such that the potentially existing + target path is first removed before the symlink is restored. https://github.com/restic/restic/issues/2578 https://github.com/restic/restic/pull/3780 * Bugfix #2591: Don't read password from stdin for `backup --stdin` - The `backup` command when used with `--stdin` previously tried to read first the password, - then the data to be backed up from standard input. This meant it would often confuse part of the - data for the password. + The `backup` command when used with `--stdin` previously tried to read first the + password, then the data to be backed up from standard input. This meant it would + often confuse part of the data for the password. - From now on, it will instead exit with the message `Fatal: cannot read both password and data - from stdin` unless the password is passed in some other way (such as - `--restic-password-file`, `RESTIC_PASSWORD`, etc). + From now on, it will instead exit with the message `Fatal: cannot read both + password and data from stdin` unless the password is passed in some other way + (such as `--restic-password-file`, `RESTIC_PASSWORD`, etc). - To enter the password interactively a password command has to be used. For example on Linux, - `mysqldump somedatabase | restic backup --stdin --password-command='sh -c - "systemd-ask-password < /dev/tty"'` securely reads the password from the terminal. + To enter the password interactively a password command has to be used. For + example on Linux, `mysqldump somedatabase | restic backup --stdin + --password-command='sh -c "systemd-ask-password < /dev/tty"'` securely reads the + password from the terminal. https://github.com/restic/restic/issues/2591 https://github.com/restic/restic/pull/4011 * Bugfix #3161: Delete files on Backblaze B2 more reliably - Restic used to only delete the latest version of files stored in B2. In most cases this worked - well as there was only a single version of the file. However, due to retries while uploading it is - possible for multiple file versions to be stored at B2. This could lead to various problems for - files that should have been deleted but still existed. + Restic used to only delete the latest version of files stored in B2. In most + cases this worked well as there was only a single version of the file. However, + due to retries while uploading it is possible for multiple file versions to be + stored at B2. This could lead to various problems for files that should have + been deleted but still existed. - The implementation has now been changed to delete all versions of files, which doubles the - amount of Class B transactions necessary to delete files, but assures that no file versions are - left behind. + The implementation has now been changed to delete all versions of files, which + doubles the amount of Class B transactions necessary to delete files, but + assures that no file versions are left behind. https://github.com/restic/restic/issues/3161 https://github.com/restic/restic/pull/3885 * Bugfix #3336: Make SFTP backend report no space left on device - Backing up to an SFTP backend would spew repeated SSH_FX_FAILURE messages when the remote disk - was full. Restic now reports "sftp: no space left on device" and exits immediately when it - detects this condition. + Backing up to an SFTP backend would spew repeated SSH_FX_FAILURE messages when + the remote disk was full. Restic now reports "sftp: no space left on device" and + exits immediately when it detects this condition. - A fix for this issue was implemented in restic 0.12.1, but unfortunately the fix itself - contained a bug that prevented it from taking effect. + A fix for this issue was implemented in restic 0.12.1, but unfortunately the fix + itself contained a bug that prevented it from taking effect. https://github.com/restic/restic/issues/3336 https://github.com/restic/restic/pull/3345 @@ -790,9 +946,10 @@ restic users. The changes are ordered by importance. * Bugfix #3567: Improve handling of interrupted syscalls in `mount` command - Accessing restic's FUSE mount could result in "input/output" errors when using programs in - which syscalls can be interrupted. This is for example the case for Go programs. This has now - been fixed by improved error handling of interrupted syscalls. + Accessing restic's FUSE mount could result in "input/output" errors when using + programs in which syscalls can be interrupted. This is for example the case for + Go programs. This has now been fixed by improved error handling of interrupted + syscalls. https://github.com/restic/restic/issues/3567 https://github.com/restic/restic/issues/3694 @@ -800,50 +957,53 @@ restic users. The changes are ordered by importance. * Bugfix #3897: Fix stuck `copy` command when `-o .connections=1` - When running the `copy` command with `-o .connections=1` the command would be - infinitely stuck. This has now been fixed. + When running the `copy` command with `-o .connections=1` the command + would be infinitely stuck. This has now been fixed. https://github.com/restic/restic/issues/3897 https://github.com/restic/restic/pull/3898 * Bugfix #3918: Correct prune statistics for partially compressed repositories - In a partially compressed repository, one data blob can exist both in an uncompressed and a - compressed version. This caused the `prune` statistics to become inaccurate and e.g. report a - too high value for the unused size, such as "unused size after prune: 16777215.991 TiB". This - has now been fixed. + In a partially compressed repository, one data blob can exist both in an + uncompressed and a compressed version. This caused the `prune` statistics to + become inaccurate and e.g. report a too high value for the unused size, such as + "unused size after prune: 16777215.991 TiB". This has now been fixed. https://github.com/restic/restic/issues/3918 https://github.com/restic/restic/pull/3980 * Bugfix #3951: Make `ls` return exit code 1 if snapshot cannot be loaded - The `ls` command used to show a warning and return exit code 0 when failing to load a snapshot. - This has now been fixed such that it instead returns exit code 1 (still showing a warning). + The `ls` command used to show a warning and return exit code 0 when failing to + load a snapshot. This has now been fixed such that it instead returns exit code + 1 (still showing a warning). https://github.com/restic/restic/pull/3951 * Bugfix #4003: Make `backup` no longer hang on Solaris when seeing a FIFO file - The `backup` command used to hang on Solaris whenever it encountered a FIFO file (named pipe), - due to a bug in the handling of extended attributes. This bug has now been fixed. + The `backup` command used to hang on Solaris whenever it encountered a FIFO file + (named pipe), due to a bug in the handling of extended attributes. This bug has + now been fixed. https://github.com/restic/restic/issues/4003 https://github.com/restic/restic/pull/4053 * Bugfix #4016: Support ExFAT-formatted local backends on macOS Ventura - ExFAT-formatted disks could not be used as local backends starting from macOS Ventura. Restic - commands would fail with an "inappropriate ioctl for device" error. This has now been fixed. + ExFAT-formatted disks could not be used as local backends starting from macOS + Ventura. Restic commands would fail with an "inappropriate ioctl for device" + error. This has now been fixed. https://github.com/restic/restic/issues/4016 https://github.com/restic/restic/pull/4021 * Bugfix #4085: Make `init` ignore "Access Denied" errors when creating S3 buckets - In restic 0.9.0 through 0.13.0, the `init` command ignored some permission errors from S3 - backends when trying to check for bucket existence, so that manually created buckets with - custom permissions could be used for backups. + In restic 0.9.0 through 0.13.0, the `init` command ignored some permission + errors from S3 backends when trying to check for bucket existence, so that + manually created buckets with custom permissions could be used for backups. This feature became broken in 0.14.0, but has now been restored again. @@ -852,20 +1012,21 @@ restic users. The changes are ordered by importance. * Bugfix #4100: Make `self-update` enabled by default only in release builds - The `self-update` command was previously included by default in all builds of restic as - opposed to only in official release builds, even if the `selfupdate` tag was not explicitly - enabled when building. + The `self-update` command was previously included by default in all builds of + restic as opposed to only in official release builds, even if the `selfupdate` + tag was not explicitly enabled when building. - This has now been corrected, and the `self-update` command is only available if restic was - built with `-tags selfupdate` (as done for official release builds by `build.go`). + This has now been corrected, and the `self-update` command is only available if + restic was built with `-tags selfupdate` (as done for official release builds by + `build.go`). https://github.com/restic/restic/pull/4100 * Bugfix #4103: Don't generate negative UIDs and GIDs in tar files from `dump` - When using a 32-bit build of restic, the `dump` command could in some cases create tar files - containing negative UIDs and GIDs, which cannot be read by GNU tar. This corner case especially - applies to backups from stdin on Windows. + When using a 32-bit build of restic, the `dump` command could in some cases + create tar files containing negative UIDs and GIDs, which cannot be read by GNU + tar. This corner case especially applies to backups from stdin on Windows. This is now fixed such that `dump` creates valid tar files in these cases too. @@ -874,48 +1035,50 @@ restic users. The changes are ordered by importance. * Change #2724: Include full snapshot ID in JSON output of `backup` - We have changed the JSON output of the backup command to include the full snapshot ID instead of - just a shortened version, as the latter can be ambiguous in some rare cases. To derive the short - ID, please truncate the full ID down to eight characters. + We have changed the JSON output of the backup command to include the full + snapshot ID instead of just a shortened version, as the latter can be ambiguous + in some rare cases. To derive the short ID, please truncate the full ID down to + eight characters. https://github.com/restic/restic/issues/2724 https://github.com/restic/restic/pull/3993 * Change #3929: Make `unlock` display message only when locks were actually removed - The `unlock` command used to print the "successfully removed locks" message whenever it was - run, regardless of lock files having being removed or not. + The `unlock` command used to print the "successfully removed locks" message + whenever it was run, regardless of lock files having being removed or not. - This has now been changed such that it only prints the message if any lock files were actually - removed. In addition, it also reports the number of removed lock files. + This has now been changed such that it only prints the message if any lock files + were actually removed. In addition, it also reports the number of removed lock + files. https://github.com/restic/restic/issues/3929 https://github.com/restic/restic/pull/3935 * Change #4033: Don't print skipped snapshots by default in `copy` command - The `copy` command used to print each snapshot that was skipped because it already existed in - the target repository. The amount of this output could practically bury the list of snapshots - that were actually copied. + The `copy` command used to print each snapshot that was skipped because it + already existed in the target repository. The amount of this output could + practically bury the list of snapshots that were actually copied. - From now on, the skipped snapshots are by default not printed at all, but this can be re-enabled - by increasing the verbosity level of the command. + From now on, the skipped snapshots are by default not printed at all, but this + can be re-enabled by increasing the verbosity level of the command. https://github.com/restic/restic/issues/4033 https://github.com/restic/restic/pull/4066 * Change #4041: Update dependencies and require Go 1.18 or newer - Most dependencies have been updated. Since some libraries require newer language features, - support for Go 1.15-1.17 has been dropped, which means that restic now requires at least Go 1.18 - to build. + Most dependencies have been updated. Since some libraries require newer language + features, support for Go 1.15-1.17 has been dropped, which means that restic now + requires at least Go 1.18 to build. https://github.com/restic/restic/pull/4041 * Enhancement #14: Implement `rewrite` command - Restic now has a `rewrite` command which allows to rewrite existing snapshots to remove - unwanted files. + Restic now has a `rewrite` command which allows to rewrite existing snapshots to + remove unwanted files. https://github.com/restic/restic/issues/14 https://github.com/restic/restic/pull/2731 @@ -923,15 +1086,15 @@ restic users. The changes are ordered by importance. * Enhancement #79: Restore files with long runs of zeros as sparse files - When using `restore --sparse`, the restorer may now write files containing long runs of zeros - as sparse files (also called files with holes), where the zeros are not actually written to - disk. + When using `restore --sparse`, the restorer may now write files containing long + runs of zeros as sparse files (also called files with holes), where the zeros + are not actually written to disk. - How much space is saved by writing sparse files depends on the operating system, file system and - the distribution of zeros in the file. + How much space is saved by writing sparse files depends on the operating system, + file system and the distribution of zeros in the file. - During backup restic still reads the whole file including sparse regions, but with optimized - processing speed of sparse regions. + During backup restic still reads the whole file including sparse regions, but + with optimized processing speed of sparse regions. https://github.com/restic/restic/issues/79 https://github.com/restic/restic/issues/3903 @@ -941,9 +1104,9 @@ restic users. The changes are ordered by importance. * Enhancement #1078: Support restoring symbolic links on Windows - The `restore` command now supports restoring symbolic links on Windows. Because of Windows - specific restrictions this is only possible when running restic with the - `SeCreateSymbolicLinkPrivilege` privilege or as an administrator. + The `restore` command now supports restoring symbolic links on Windows. Because + of Windows specific restrictions this is only possible when running restic with + the `SeCreateSymbolicLinkPrivilege` privilege or as an administrator. https://github.com/restic/restic/issues/1078 https://github.com/restic/restic/issues/2699 @@ -951,14 +1114,14 @@ restic users. The changes are ordered by importance. * Enhancement #1734: Inform about successful retries after errors - When a recoverable error is encountered, restic shows a warning message saying that it's - retrying, e.g.: + When a recoverable error is encountered, restic shows a warning message saying + that it's retrying, e.g.: `Save() returned error, retrying after 357.131936ms: ...` - This message can be confusing in that it never clearly states whether the retry is successful or - not. This has now been fixed such that restic follows up with a message confirming a successful - retry, e.g.: + This message can be confusing in that it never clearly states whether the retry + is successful or not. This has now been fixed such that restic follows up with a + message confirming a successful retry, e.g.: `Save() operation successful after 1 retries` @@ -967,12 +1130,12 @@ restic users. The changes are ordered by importance. * Enhancement #1866: Improve handling of directories with duplicate entries - If for some reason a directory contains a duplicate entry, the `backup` command would - previously fail with a `node "path/to/file" already present` or `nodes are not ordered got - "path/to/file", last "path/to/file"` error. + If for some reason a directory contains a duplicate entry, the `backup` command + would previously fail with a `node "path/to/file" already present` or `nodes are + not ordered got "path/to/file", last "path/to/file"` error. - The error handling has been improved to only report a warning in this case. Make sure to check - that the filesystem in question is not damaged if you see this! + The error handling has been improved to only report a warning in this case. Make + sure to check that the filesystem in question is not damaged if you see this! https://github.com/restic/restic/issues/1866 https://github.com/restic/restic/issues/3937 @@ -980,29 +1143,31 @@ restic users. The changes are ordered by importance. * Enhancement #2134: Support B2 API keys restricted to hiding but not deleting files - When the B2 backend does not have the necessary permissions to permanently delete files, it now - automatically falls back to hiding files. This allows using restic with an application key - which is not allowed to delete files. This can prevent an attacker from deleting backups with - such an API key. + When the B2 backend does not have the necessary permissions to permanently + delete files, it now automatically falls back to hiding files. This allows using + restic with an application key which is not allowed to delete files. This can + prevent an attacker from deleting backups with such an API key. - To use this feature create an application key without the `deleteFiles` capability. It is - recommended to restrict the key to just one bucket. For example using the `b2` command line - tool: + To use this feature create an application key without the `deleteFiles` + capability. It is recommended to restrict the key to just one bucket. For + example using the `b2` command line tool: `b2 create-key --bucket listBuckets,readFiles,writeFiles,listFiles` - Alternatively, you can use the S3 backend to access B2, as described in the documentation. In - this mode, files are also only hidden instead of being deleted permanently. + Alternatively, you can use the S3 backend to access B2, as described in the + documentation. In this mode, files are also only hidden instead of being deleted + permanently. https://github.com/restic/restic/issues/2134 https://github.com/restic/restic/pull/2398 * Enhancement #2152: Make `init` open only one connection for the SFTP backend - The `init` command using the SFTP backend used to connect twice to the repository. This could be - inconvenient if the user must enter a password, or cause `init` to fail if the server does not - correctly close the first SFTP connection. + The `init` command using the SFTP backend used to connect twice to the + repository. This could be inconvenient if the user must enter a password, or + cause `init` to fail if the server does not correctly close the first SFTP + connection. This has now been fixed by reusing the first/initial SFTP connection opened. @@ -1011,40 +1176,44 @@ restic users. The changes are ordered by importance. * Enhancement #2533: Handle cache corruption on disk and in downloads - In rare situations, like for example after a system crash, the data stored in the cache might be - corrupted. This could cause restic to fail and required manually deleting the cache. + In rare situations, like for example after a system crash, the data stored in + the cache might be corrupted. This could cause restic to fail and required + manually deleting the cache. - Restic now automatically removes broken data from the cache, allowing it to recover from such a - situation without user intervention. In addition, restic retries downloads which return - corrupt data in order to also handle temporary download problems. + Restic now automatically removes broken data from the cache, allowing it to + recover from such a situation without user intervention. In addition, restic + retries downloads which return corrupt data in order to also handle temporary + download problems. https://github.com/restic/restic/issues/2533 https://github.com/restic/restic/pull/3521 * Enhancement #2715: Stricter repository lock handling - Previously, restic commands kept running even if they failed to refresh their locks in time. - This could be a problem e.g. in case the client system running a backup entered the standby power - mode while the backup was still in progress (which would prevent the client from refreshing its - lock), and after a short delay another host successfully runs `unlock` and `prune` on the - repository, which would remove all data added by the in-progress backup. If the backup client - later continues its backup, even though its lock had expired in the meantime, this would lead to - an incomplete snapshot. + Previously, restic commands kept running even if they failed to refresh their + locks in time. This could be a problem e.g. in case the client system running a + backup entered the standby power mode while the backup was still in progress + (which would prevent the client from refreshing its lock), and after a short + delay another host successfully runs `unlock` and `prune` on the repository, + which would remove all data added by the in-progress backup. If the backup + client later continues its backup, even though its lock had expired in the + meantime, this would lead to an incomplete snapshot. - To address this, lock handling is now much stricter. Commands requiring a lock are canceled if - the lock is not refreshed successfully in time. In addition, if a lock file is not readable - restic will not allow starting a command. It may be necessary to remove invalid lock files - manually or use `unlock --remove-all`. Please make sure that no other restic processes are - running concurrently before doing this, however. + To address this, lock handling is now much stricter. Commands requiring a lock + are canceled if the lock is not refreshed successfully in time. In addition, if + a lock file is not readable restic will not allow starting a command. It may be + necessary to remove invalid lock files manually or use `unlock --remove-all`. + Please make sure that no other restic processes are running concurrently before + doing this, however. https://github.com/restic/restic/issues/2715 https://github.com/restic/restic/pull/3569 * Enhancement #2750: Make backup file read concurrency configurable - The `backup` command now supports a `--read-concurrency` option which allows tuning restic - for very fast storage like NVMe disks by controlling the number of concurrent file reads during - the backup process. + The `backup` command now supports a `--read-concurrency` option which allows + tuning restic for very fast storage like NVMe disks by controlling the number of + concurrent file reads during the backup process. https://github.com/restic/restic/pull/2750 @@ -1059,75 +1228,78 @@ restic users. The changes are ordered by importance. * Enhancement #3096: Make `mount` command support macOS using macFUSE 4.x - Restic now uses a different FUSE library for mounting snapshots and making them available as a - FUSE filesystem using the `mount` command. This adds support for macFUSE 4.x which can be used - to make this work on recent macOS versions. + Restic now uses a different FUSE library for mounting snapshots and making them + available as a FUSE filesystem using the `mount` command. This adds support for + macFUSE 4.x which can be used to make this work on recent macOS versions. https://github.com/restic/restic/issues/3096 https://github.com/restic/restic/pull/4024 * Enhancement #3124: Support JSON output for the `init` command - The `init` command used to ignore the `--json` option, but now outputs a JSON message if the - repository was created successfully. + The `init` command used to ignore the `--json` option, but now outputs a JSON + message if the repository was created successfully. https://github.com/restic/restic/issues/3124 https://github.com/restic/restic/pull/3132 * Enhancement #3899: Optimize prune memory usage - The `prune` command needs large amounts of memory in order to determine what to keep and what to - remove. This is now optimized to use up to 30% less memory. + The `prune` command needs large amounts of memory in order to determine what to + keep and what to remove. This is now optimized to use up to 30% less memory. https://github.com/restic/restic/pull/3899 * Enhancement #3905: Improve speed of parent snapshot detection in `backup` command - Backing up a large number of files using `--files-from-verbatim` or `--files-from-raw` - options could require a long time to find the parent snapshot. This has been improved. + Backing up a large number of files using `--files-from-verbatim` or + `--files-from-raw` options could require a long time to find the parent + snapshot. This has been improved. https://github.com/restic/restic/pull/3905 * Enhancement #3915: Add compression statistics to the `stats` command - When executed with `--mode raw-data` on a repository that supports compression, the `stats` - command now calculates and displays, for the selected repository or snapshots: the - uncompressed size of the data; the compression progress (percentage of data that has been - compressed); the compression ratio of the compressed data; the total space saving. + When executed with `--mode raw-data` on a repository that supports compression, + the `stats` command now calculates and displays, for the selected repository or + snapshots: the uncompressed size of the data; the compression progress + (percentage of data that has been compressed); the compression ratio of the + compressed data; the total space saving. - It also takes into account both the compressed and uncompressed data if the repository is only - partially compressed. + It also takes into account both the compressed and uncompressed data if the + repository is only partially compressed. https://github.com/restic/restic/pull/3915 * Enhancement #3925: Provide command completion for PowerShell - Restic already provided generation of completion files for bash, fish and zsh. Now powershell - is supported, too. + Restic already provided generation of completion files for bash, fish and zsh. + Now powershell is supported, too. https://github.com/restic/restic/pull/3925/files * Enhancement #3931: Allow `backup` file tree scanner to be disabled - The `backup` command walks the file tree in a separate scanner process to find the total size and - file/directory count, and uses this to provide an ETA. This can slow down backups, especially - of network filesystems. + The `backup` command walks the file tree in a separate scanner process to find + the total size and file/directory count, and uses this to provide an ETA. This + can slow down backups, especially of network filesystems. - The command now has a new option `--no-scan` which can be used to disable this scanning in order - to speed up backups when needed. + The command now has a new option `--no-scan` which can be used to disable this + scanning in order to speed up backups when needed. https://github.com/restic/restic/pull/3931 * Enhancement #3932: Improve handling of ErrDot errors in rclone and sftp backends - Since Go 1.19, restic can no longer implicitly run relative executables which are found in the - current directory (e.g. `rclone` if found in `.`). This is a security feature of Go to prevent - against running unintended and possibly harmful executables. + Since Go 1.19, restic can no longer implicitly run relative executables which + are found in the current directory (e.g. `rclone` if found in `.`). This is a + security feature of Go to prevent against running unintended and possibly + harmful executables. - The error message for this was just "cannot run executable found relative to current - directory". This has now been improved to yield a more specific error message, informing the - user how to explicitly allow running the executable using the `-o rclone.program` and `-o - sftp.command` extended options with `./`. + The error message for this was just "cannot run executable found relative to + current directory". This has now been improved to yield a more specific error + message, informing the user how to explicitly allow running the executable using + the `-o rclone.program` and `-o sftp.command` extended options with `./`. https://github.com/restic/restic/issues/3932 https://pkg.go.dev/os/exec#hdr-Executables_in_the_current_directory @@ -1135,20 +1307,21 @@ restic users. The changes are ordered by importance. * Enhancement #3943: Ignore additional/unknown files in repository - If a restic repository had additional files in it (not created by restic), commands like `find` - and `restore` could become confused and fail with an `multiple IDs with prefix "12345678" - found` error. These commands now ignore such additional files. + If a restic repository had additional files in it (not created by restic), + commands like `find` and `restore` could become confused and fail with an + `multiple IDs with prefix "12345678" found` error. These commands now ignore + such additional files. https://github.com/restic/restic/pull/3943 https://forum.restic.net/t/which-protocol-should-i-choose-for-remote-linux-backups/5446/17 * Enhancement #3955: Improve `backup` performance for small files - When backing up small files restic was slower than it could be. In particular this affected - backups using maximum compression. + When backing up small files restic was slower than it could be. In particular + this affected backups using maximum compression. - This has been fixed by reworking the internal parallelism of the backup command, making it back - up small files around two times faster. + This has been fixed by reworking the internal parallelism of the backup command, + making it back up small files around two times faster. https://github.com/restic/restic/pull/3955 @@ -1197,22 +1370,23 @@ restic users. The changes are ordered by importance. * Bugfix #2248: Support `self-update` on Windows - Restic `self-update` would fail in situations where the operating system locks running - binaries, including Windows. The new behavior works around this by renaming the running file - and swapping the updated file in place. + Restic `self-update` would fail in situations where the operating system locks + running binaries, including Windows. The new behavior works around this by + renaming the running file and swapping the updated file in place. https://github.com/restic/restic/issues/2248 https://github.com/restic/restic/pull/3675 * Bugfix #3428: List snapshots in backend at most once to resolve snapshot IDs - Many commands support specifying a list of snapshot IDs which are then used to determine the - snapshots to be processed by the command. To resolve snapshot IDs or `latest`, and check that - these exist, restic previously listed all snapshots stored in the repository. Depending on - the backend this could be a slow and/or expensive operation. + Many commands support specifying a list of snapshot IDs which are then used to + determine the snapshots to be processed by the command. To resolve snapshot IDs + or `latest`, and check that these exist, restic previously listed all snapshots + stored in the repository. Depending on the backend this could be a slow and/or + expensive operation. - Restic now lists the snapshots only once and remembers the result in order to resolve all - further snapshot IDs swiftly. + Restic now lists the snapshots only once and remembers the result in order to + resolve all further snapshot IDs swiftly. https://github.com/restic/restic/issues/3428 https://github.com/restic/restic/pull/3570 @@ -1220,27 +1394,28 @@ restic users. The changes are ordered by importance. * Bugfix #3432: Fix rare 'not found in repository' error for `copy` command - In rare cases `copy` (and other commands) would report that `LoadTree(...)` returned an `id - [...] not found in repository` error. This could be caused by a backup or copy command running - concurrently. The error was only temporary; running the failed restic command a second time as - a workaround did resolve the error. + In rare cases `copy` (and other commands) would report that `LoadTree(...)` + returned an `id [...] not found in repository` error. This could be caused by a + backup or copy command running concurrently. The error was only temporary; + running the failed restic command a second time as a workaround did resolve the + error. - This issue has now been fixed by correcting the order in which restic reads data from the - repository. It is now guaranteed that restic only loads snapshots for which all necessary data - is already available. + This issue has now been fixed by correcting the order in which restic reads data + from the repository. It is now guaranteed that restic only loads snapshots for + which all necessary data is already available. https://github.com/restic/restic/issues/3432 https://github.com/restic/restic/pull/3570 * Bugfix #3681: Fix rclone (shimmed by Scoop) and sftp not working on Windows - In #3602 a fix was introduced to address the problem of `rclone` prematurely exiting when - Ctrl+C is pressed on Windows. The solution was to create the subprocess with its console - detached from the restic console. + In #3602 a fix was introduced to address the problem of `rclone` prematurely + exiting when Ctrl+C is pressed on Windows. The solution was to create the + subprocess with its console detached from the restic console. - However, this solution failed when using `rclone` installed by Scoop or using `sftp` with a - passphrase-protected private key. We've now fixed this by using a different approach to - prevent Ctrl-C from passing down too early. + However, this solution failed when using `rclone` installed by Scoop or using + `sftp` with a passphrase-protected private key. We've now fixed this by using a + different approach to prevent Ctrl-C from passing down too early. https://github.com/restic/restic/issues/3681 https://github.com/restic/restic/issues/3692 @@ -1248,28 +1423,28 @@ restic users. The changes are ordered by importance. * Bugfix #3685: The `diff` command incorrectly listed some files as added - There was a bug in the `diff` command, causing it to always show files in a removed directory as - added. This has now been fixed. + There was a bug in the `diff` command, causing it to always show files in a + removed directory as added. This has now been fixed. https://github.com/restic/restic/issues/3685 https://github.com/restic/restic/pull/3686 * Bugfix #3716: Print "wrong password" to stderr instead of stdout - If an invalid password was entered, the error message was printed on stdout and not on stderr as - intended. This has now been fixed. + If an invalid password was entered, the error message was printed on stdout and + not on stderr as intended. This has now been fixed. https://github.com/restic/restic/pull/3716 https://forum.restic.net/t/4965 * Bugfix #3720: Directory sync errors for repositories accessed via SMB - On Linux and macOS, accessing a repository via a SMB/CIFS mount resulted in restic failing to - save the lock file, yielding the following errors: + On Linux and macOS, accessing a repository via a SMB/CIFS mount resulted in + restic failing to save the lock file, yielding the following errors: - Save() returned error, retrying after 552.330144ms: sync /repo/locks: - no such file or directory Save() returned error, retrying after - 552.330144ms: sync /repo/locks: invalid argument + Save() returned error, retrying after 552.330144ms: sync + /repo/locks: no such file or directory Save() returned error, + retrying after 552.330144ms: sync /repo/locks: invalid argument This has now been fixed by ignoring the relevant error codes. @@ -1279,22 +1454,23 @@ restic users. The changes are ordered by importance. * Bugfix #3736: The `stats` command miscalculated restore size for multiple snapshots - Since restic 0.10.0 the restore size calculated by the `stats` command for multiple snapshots - was too low. The hardlink detection was accidentally applied across multiple snapshots and - thus ignored many files. This has now been fixed. + Since restic 0.10.0 the restore size calculated by the `stats` command for + multiple snapshots was too low. The hardlink detection was accidentally applied + across multiple snapshots and thus ignored many files. This has now been fixed. https://github.com/restic/restic/issues/3736 https://github.com/restic/restic/pull/3740 * Bugfix #3772: Correctly rebuild index for legacy repositories - After running `rebuild-index` on a legacy repository containing mixed pack files (that is, - pack files which store both metadata and file data), `check` printed warnings like `pack - 12345678 contained in several indexes: ...`. This warning was not critical, but has now - nonetheless been fixed by properly handling mixed pack files while rebuilding the index. + After running `rebuild-index` on a legacy repository containing mixed pack files + (that is, pack files which store both metadata and file data), `check` printed + warnings like `pack 12345678 contained in several indexes: ...`. This warning + was not critical, but has now nonetheless been fixed by properly handling mixed + pack files while rebuilding the index. - Running `prune` for such legacy repositories will also fix the warning by reorganizing the - pack files which caused it. + Running `prune` for such legacy repositories will also fix the warning by + reorganizing the pack files which caused it. https://github.com/restic/restic/pull/3772 https://github.com/restic/restic/pull/3884 @@ -1302,18 +1478,20 @@ restic users. The changes are ordered by importance. * Bugfix #3776: Limit number of key files tested while opening a repository - Previously, restic tested the password against every key in the repository when opening a - repository. The more keys there were in the repository, the slower this operation became. + Previously, restic tested the password against every key in the repository when + opening a repository. The more keys there were in the repository, the slower + this operation became. - Restic now tests the password against up to 20 key files in the repository. Alternatively, you - can use the `--key-hint=` option to specify a specific key file to use instead. + Restic now tests the password against up to 20 key files in the repository. + Alternatively, you can use the `--key-hint=` option to specify a + specific key file to use instead. https://github.com/restic/restic/pull/3776 * Bugfix #3861: Yield error on invalid policy to `forget` - The `forget` command previously silently ignored invalid/unsupported units in the duration - options, such as e.g. `--keep-within-daily 2w`. + The `forget` command previously silently ignored invalid/unsupported units in + the duration options, such as e.g. `--keep-within-daily 2w`. Specifying an invalid/unsupported duration unit now results in an error. @@ -1322,71 +1500,78 @@ restic users. The changes are ordered by importance. * Change #1842: Support debug log creation in release builds - Creating a debug log was only possible in debug builds which required users to manually build - restic. We changed the release builds to allow creating debug logs by simply setting the - environment variable `DEBUG_LOG=logname.log`. + Creating a debug log was only possible in debug builds which required users to + manually build restic. We changed the release builds to allow creating debug + logs by simply setting the environment variable `DEBUG_LOG=logname.log`. https://github.com/restic/restic/issues/1842 https://github.com/restic/restic/pull/3826 * Change #3295: Deprecate `check --check-unused` and add further checks - Since restic 0.12.0, it is expected to still have unused blobs after running `prune`. This made - the `--check-unused` option of the `check` command rather useless and tended to confuse - users. This option has been deprecated and is now ignored. + Since restic 0.12.0, it is expected to still have unused blobs after running + `prune`. This made the `--check-unused` option of the `check` command rather + useless and tended to confuse users. This option has been deprecated and is now + ignored. - The `check` command now also warns if a repository is using either the legacy S3 layout or mixed - pack files with both tree and data blobs. The latter is known to cause performance problems. + The `check` command now also warns if a repository is using either the legacy S3 + layout or mixed pack files with both tree and data blobs. The latter is known to + cause performance problems. https://github.com/restic/restic/issues/3295 https://github.com/restic/restic/pull/3730 * Change #3680: Update dependencies and require Go 1.15 or newer - We've updated most dependencies. Since some libraries require newer language features we're - dropping support for Go 1.14, which means that restic now requires at least Go 1.15 to build. + We've updated most dependencies. Since some libraries require newer language + features we're dropping support for Go 1.14, which means that restic now + requires at least Go 1.15 to build. https://github.com/restic/restic/issues/3680 https://github.com/restic/restic/issues/3883 * Change #3742: Replace `--repo2` option used by `init`/`copy` with `--from-repo` - The `init` and `copy` commands can read data from another repository. However, confusingly - `--repo2` referred to the repository *from* which the `init` command copies parameters, but - for the `copy` command `--repo2` referred to the copy *destination*. + The `init` and `copy` commands can read data from another repository. However, + confusingly `--repo2` referred to the repository *from* which the `init` command + copies parameters, but for the `copy` command `--repo2` referred to the copy + *destination*. - We've introduced a new option, `--from-repo`, which always refers to the source repository - for both commands. The old parameter names have been deprecated but still work. To create a new - repository and copy all snapshots to it, the commands are now as follows: + We've introduced a new option, `--from-repo`, which always refers to the source + repository for both commands. The old parameter names have been deprecated but + still work. To create a new repository and copy all snapshots to it, the + commands are now as follows: - ``` restic -r /srv/restic-repo-copy init --from-repo /srv/restic-repo - --copy-chunker-params restic -r /srv/restic-repo-copy copy --from-repo - /srv/restic-repo ``` + ``` + restic -r /srv/restic-repo-copy init --from-repo /srv/restic-repo --copy-chunker-params + restic -r /srv/restic-repo-copy copy --from-repo /srv/restic-repo + ``` https://github.com/restic/restic/pull/3742 https://forum.restic.net/t/5017 * Enhancement #21: Add compression support - We've added compression support to the restic repository format. To create a repository using - the new format run `init --repository-version 2`. Please note that the repository cannot be - read by restic versions prior to 0.14.0. + We've added compression support to the restic repository format. To create a + repository using the new format run `init --repository-version 2`. Please note + that the repository cannot be read by restic versions prior to 0.14.0. - You can configure whether data is compressed with the option `--compression`. It can be set to - `auto` (the default, which will compress very fast), `max` (which will trade backup speed and - CPU usage for better compression), or `off` (which disables compression). Each setting is - only applied for the current run of restic and does *not* apply to future runs. The option can - also be set via the environment variable `RESTIC_COMPRESSION`. + You can configure whether data is compressed with the option `--compression`. It + can be set to `auto` (the default, which will compress very fast), `max` (which + will trade backup speed and CPU usage for better compression), or `off` (which + disables compression). Each setting is only applied for the current run of + restic and does *not* apply to future runs. The option can also be set via the + environment variable `RESTIC_COMPRESSION`. - To upgrade in place run `migrate upgrade_repo_v2` followed by `prune`. See the documentation - for more details. The migration checks the repository integrity and upgrades the repository - format, but will not change any data. Afterwards, prune will rewrite the metadata to make use of - compression. + To upgrade in place run `migrate upgrade_repo_v2` followed by `prune`. See the + documentation for more details. The migration checks the repository integrity + and upgrades the repository format, but will not change any data. Afterwards, + prune will rewrite the metadata to make use of compression. - As an alternative you can use the `copy` command to migrate snapshots; First create a new - repository using `init --repository-version 2 --copy-chunker-params --repo2 - path/to/old/repo`, and then use the `copy` command to copy all snapshots to the new - repository. + As an alternative you can use the `copy` command to migrate snapshots; First + create a new repository using `init --repository-version 2 --copy-chunker-params + --repo2 path/to/old/repo`, and then use the `copy` command to copy all snapshots + to the new repository. https://github.com/restic/restic/issues/21 https://github.com/restic/restic/issues/3779 @@ -1396,25 +1581,28 @@ restic users. The changes are ordered by importance. * Enhancement #1153: Support pruning even when the disk is full - When running out of disk space it was no longer possible to add or remove data from a repository. - To help with recovering from such a deadlock, the prune command now supports an - `--unsafe-recover-no-free-space` option to recover from these situations. Make sure to - read the documentation first! + When running out of disk space it was no longer possible to add or remove data + from a repository. To help with recovering from such a deadlock, the prune + command now supports an `--unsafe-recover-no-free-space` option to recover from + these situations. Make sure to read the documentation first! https://github.com/restic/restic/issues/1153 https://github.com/restic/restic/pull/3481 * Enhancement #2162: Adaptive IO concurrency based on backend connections - Many commands used hard-coded limits for the number of concurrent operations. This prevented - speed improvements by increasing the number of connections used by a backend. + Many commands used hard-coded limits for the number of concurrent operations. + This prevented speed improvements by increasing the number of connections used + by a backend. - These limits have now been replaced by using the configured number of backend connections - instead, which can be controlled using the `-o .connections=5` option. - Commands will then automatically scale their parallelism accordingly. + These limits have now been replaced by using the configured number of backend + connections instead, which can be controlled using the `-o + .connections=5` option. Commands will then automatically scale + their parallelism accordingly. - To limit the number of CPU cores used by restic, you can set the environment variable - `GOMAXPROCS` accordingly. For example to use a single CPU core, use `GOMAXPROCS=1`. + To limit the number of CPU cores used by restic, you can set the environment + variable `GOMAXPROCS` accordingly. For example to use a single CPU core, use + `GOMAXPROCS=1`. https://github.com/restic/restic/issues/2162 https://github.com/restic/restic/issues/1467 @@ -1422,45 +1610,47 @@ restic users. The changes are ordered by importance. * Enhancement #2291: Allow pack size customization - Restic now uses a target pack size of 16 MiB by default. This can be customized using the - `--pack-size size` option. Supported pack sizes range between 4 and 128 MiB. + Restic now uses a target pack size of 16 MiB by default. This can be customized + using the `--pack-size size` option. Supported pack sizes range between 4 and + 128 MiB. - It is possible to migrate an existing repository to _larger_ pack files using `prune - --repack-small`. This will rewrite every pack file which is significantly smaller than the - target size. + It is possible to migrate an existing repository to _larger_ pack files using + `prune --repack-small`. This will rewrite every pack file which is significantly + smaller than the target size. https://github.com/restic/restic/issues/2291 https://github.com/restic/restic/pull/3731 * Enhancement #2295: Allow use of SAS token to authenticate to Azure - Previously restic only supported AccountKeys to authenticate to Azure storage accounts, - which necessitates giving a significant amount of access. + Previously restic only supported AccountKeys to authenticate to Azure storage + accounts, which necessitates giving a significant amount of access. - We added support for Azure SAS tokens which are a more fine-grained and time-limited manner of - granting access. Set the `AZURE_ACCOUNT_NAME` and `AZURE_ACCOUNT_SAS` environment - variables to use a SAS token for authentication. Note that if `AZURE_ACCOUNT_KEY` is set, it - will take precedence. + We added support for Azure SAS tokens which are a more fine-grained and + time-limited manner of granting access. Set the `AZURE_ACCOUNT_NAME` and + `AZURE_ACCOUNT_SAS` environment variables to use a SAS token for authentication. + Note that if `AZURE_ACCOUNT_KEY` is set, it will take precedence. https://github.com/restic/restic/issues/2295 https://github.com/restic/restic/pull/3661 * Enhancement #2351: Use config file permissions to control file group access - Previously files in a local/SFTP repository would always end up with very restrictive access - permissions, allowing access only to the owner. This prevented a number of valid use-cases - involving groups and ACLs. + Previously files in a local/SFTP repository would always end up with very + restrictive access permissions, allowing access only to the owner. This + prevented a number of valid use-cases involving groups and ACLs. - We now use the permissions of the config file in the repository to decide whether group access - should be given to newly created repository files or not. We arrange for repository files to be - created group readable exactly when the repository config file is group readable. + We now use the permissions of the config file in the repository to decide + whether group access should be given to newly created repository files or not. + We arrange for repository files to be created group readable exactly when the + repository config file is group readable. - To opt-in to group readable repositories, a simple `chmod -R g+r` or equivalent on the config - file can be used. For repositories that should be writable by group members a tad more setup is - required, see the docs. + To opt-in to group readable repositories, a simple `chmod -R g+r` or equivalent + on the config file can be used. For repositories that should be writable by + group members a tad more setup is required, see the docs. - Posix ACLs can also be used now that the group permissions being forced to zero no longer masks - the effect of ACL entries. + Posix ACLs can also be used now that the group permissions being forced to zero + no longer masks the effect of ACL entries. https://github.com/restic/restic/issues/2351 https://github.com/restic/restic/pull/3419 @@ -1468,27 +1658,29 @@ restic users. The changes are ordered by importance. * Enhancement #2696: Improve backup speed with many small files - We have restructured the backup pipeline to continue reading files while all upload - connections are busy. This allows the backup to already prepare the next data file such that the - upload can continue as soon as a connection becomes available. This can especially improve the - backup performance for high latency backends. + We have restructured the backup pipeline to continue reading files while all + upload connections are busy. This allows the backup to already prepare the next + data file such that the upload can continue as soon as a connection becomes + available. This can especially improve the backup performance for high latency + backends. - The upload concurrency is now controlled using the `-o .connections=5` - option. + The upload concurrency is now controlled using the `-o + .connections=5` option. https://github.com/restic/restic/issues/2696 https://github.com/restic/restic/pull/3489 * Enhancement #2907: Make snapshot directory structure of `mount` command customizable - We've added the possibility to customize the snapshot directory structure of the `mount` - command using templates passed to the `--snapshot-template` option. The formatting of - snapshots' timestamps is now controlled using `--time-template` and supports - subdirectories to for example group snapshots by year. Please see `restic help mount` for - further details. + We've added the possibility to customize the snapshot directory structure of the + `mount` command using templates passed to the `--snapshot-template` option. The + formatting of snapshots' timestamps is now controlled using `--time-template` + and supports subdirectories to for example group snapshots by year. Please see + `restic help mount` for further details. - Characters in tag names which are not allowed in a filename are replaced by underscores `_`. For - example a tag `foo/bar` will result in a directory name of `foo_bar`. + Characters in tag names which are not allowed in a filename are replaced by + underscores `_`. For example a tag `foo/bar` will result in a directory name of + `foo_bar`. https://github.com/restic/restic/issues/2907 https://github.com/restic/restic/pull/2913 @@ -1496,8 +1688,9 @@ restic users. The changes are ordered by importance. * Enhancement #2923: Improve speed of `copy` command - The `copy` command could require a long time to copy snapshots for non-local backends. This has - been improved to provide a throughput comparable to the `restore` command. + The `copy` command could require a long time to copy snapshots for non-local + backends. This has been improved to provide a throughput comparable to the + `restore` command. Additionally, `copy` now displays a progress bar. @@ -1506,21 +1699,23 @@ restic users. The changes are ordered by importance. * Enhancement #3114: Optimize handling of duplicate blobs in `prune` - Restic `prune` always used to repack all data files containing duplicate blobs. This - effectively removed all duplicates during prune. However, as a consequence all these data - files were repacked even if the unused repository space threshold could be reached with less - work. + Restic `prune` always used to repack all data files containing duplicate blobs. + This effectively removed all duplicates during prune. However, as a consequence + all these data files were repacked even if the unused repository space threshold + could be reached with less work. - This is now changed and `prune` works nice and fast even when there are lots of duplicate blobs. + This is now changed and `prune` works nice and fast even when there are lots of + duplicate blobs. https://github.com/restic/restic/issues/3114 https://github.com/restic/restic/pull/3290 * Enhancement #3465: Improve handling of temporary files on Windows - In some cases restic failed to delete temporary files, causing the current command to fail. - This has now been fixed by ensuring that Windows automatically deletes the file. In addition, - temporary files are only written to disk when necessary, reducing disk writes. + In some cases restic failed to delete temporary files, causing the current + command to fail. This has now been fixed by ensuring that Windows automatically + deletes the file. In addition, temporary files are only written to disk when + necessary, reducing disk writes. https://github.com/restic/restic/issues/3465 https://github.com/restic/restic/issues/1551 @@ -1528,22 +1723,23 @@ restic users. The changes are ordered by importance. * Enhancement #3475: Allow limiting IO concurrency for local and SFTP backend - Restic did not support limiting the IO concurrency / number of connections for accessing - repositories stored using the local or SFTP backends. The number of connections is now limited - as for other backends, and can be configured via the `-o local.connections=2` and `-o - sftp.connections=5` options. This ensures that restic does not overwhelm the backend with - concurrent IO operations. + Restic did not support limiting the IO concurrency / number of connections for + accessing repositories stored using the local or SFTP backends. The number of + connections is now limited as for other backends, and can be configured via the + `-o local.connections=2` and `-o sftp.connections=5` options. This ensures that + restic does not overwhelm the backend with concurrent IO operations. https://github.com/restic/restic/pull/3475 * Enhancement #3484: Stream data in `check` and `prune` commands - The commands `check --read-data` and `prune` previously downloaded data files into - temporary files which could end up being written to disk. This could cause a large amount of data - being written to disk. + The commands `check --read-data` and `prune` previously downloaded data files + into temporary files which could end up being written to disk. This could cause + a large amount of data being written to disk. - The pack files are now instead streamed, which removes the need for temporary files. Please - note that *uploads* during `backup` and `prune` still require temporary files. + The pack files are now instead streamed, which removes the need for temporary + files. Please note that *uploads* during `backup` and `prune` still require + temporary files. https://github.com/restic/restic/issues/3710 https://github.com/restic/restic/pull/3484 @@ -1552,19 +1748,19 @@ restic users. The changes are ordered by importance. * Enhancement #3709: Validate exclude patterns before backing up Exclude patterns provided via `--exclude`, `--iexclude`, `--exclude-file` or - `--iexclude-file` previously weren't validated. As a consequence, invalid patterns - resulted in files that were meant to be excluded being backed up. + `--iexclude-file` previously weren't validated. As a consequence, invalid + patterns resulted in files that were meant to be excluded being backed up. - Restic now validates all patterns before running the backup and aborts with a fatal error if an - invalid pattern is detected. + Restic now validates all patterns before running the backup and aborts with a + fatal error if an invalid pattern is detected. https://github.com/restic/restic/issues/3709 https://github.com/restic/restic/pull/3734 * Enhancement #3729: Display full IDs in `check` warnings - When running commands to inspect or repair a damaged repository, it is often necessary to - supply the full IDs of objects stored in the repository. + When running commands to inspect or repair a damaged repository, it is often + necessary to supply the full IDs of objects stored in the repository. The output of `check` now includes full IDs instead of their shortened variant. @@ -1572,28 +1768,29 @@ restic users. The changes are ordered by importance. * Enhancement #3773: Optimize memory usage for directories with many files - Backing up a directory with hundreds of thousands or more files caused restic to require large - amounts of memory. We've now optimized the `backup` command such that it requires up to 30% less - memory. + Backing up a directory with hundreds of thousands or more files caused restic to + require large amounts of memory. We've now optimized the `backup` command such + that it requires up to 30% less memory. https://github.com/restic/restic/pull/3773 * Enhancement #3819: Validate include/exclude patterns before restoring Patterns provided to `restore` via `--exclude`, `--iexclude`, `--include` and - `--iinclude` weren't validated before running the restore. Invalid patterns would result in - error messages being printed repeatedly, and possibly unwanted files being restored. + `--iinclude` weren't validated before running the restore. Invalid patterns + would result in error messages being printed repeatedly, and possibly unwanted + files being restored. - Restic now validates all patterns before running the restore, and aborts with a fatal error if - an invalid pattern is detected. + Restic now validates all patterns before running the restore, and aborts with a + fatal error if an invalid pattern is detected. https://github.com/restic/restic/pull/3819 * Enhancement #3837: Improve SFTP repository initialization over slow links - The `init` command, when used on an SFTP backend, now sends multiple `mkdir` commands to the - backend concurrently. This reduces the waiting times when creating a repository over a very - slow connection. + The `init` command, when used on an SFTP backend, now sends multiple `mkdir` + commands to the backend concurrently. This reduces the waiting times when + creating a repository over a very slow connection. https://github.com/restic/restic/issues/3837 https://github.com/restic/restic/pull/3840 @@ -1644,9 +1841,9 @@ restic users. The changes are ordered by importance. * Bugfix #1106: Never lock repository for `list locks` - The `list locks` command previously locked to the repository by default. This had the problem - that it wouldn't work for an exclusively locked repository and that the command would also - display its own lock file which can be confusing. + The `list locks` command previously locked to the repository by default. This + had the problem that it wouldn't work for an exclusively locked repository and + that the command would also display its own lock file which can be confusing. Now, the `list locks` command never locks the repository. @@ -1655,22 +1852,24 @@ restic users. The changes are ordered by importance. * Bugfix #2345: Make cache crash-resistant and usable by multiple concurrent processes - The restic cache directory (`RESTIC_CACHE_DIR`) could end up in a broken state in the event of - restic (or the OS) crashing. This is now less likely to occur as files are downloaded to a - temporary location before being moved to their proper location. + The restic cache directory (`RESTIC_CACHE_DIR`) could end up in a broken state + in the event of restic (or the OS) crashing. This is now less likely to occur as + files are downloaded to a temporary location before being moved to their proper + location. - This also allows multiple concurrent restic processes to operate on a single repository - without conflicts. Previously, concurrent operations could cause segfaults because the - processes saw each other's partially downloaded files. + This also allows multiple concurrent restic processes to operate on a single + repository without conflicts. Previously, concurrent operations could cause + segfaults because the processes saw each other's partially downloaded files. https://github.com/restic/restic/issues/2345 https://github.com/restic/restic/pull/2838 * Bugfix #2452: Improve error handling of repository locking - Previously, when the lock refresh failed to delete the old lock file, it forgot about the newly - created one. Instead it continued trying to delete the old (usually no longer existing) lock - file and thus over time lots of lock files accumulated. This has now been fixed. + Previously, when the lock refresh failed to delete the old lock file, it forgot + about the newly created one. Instead it continued trying to delete the old + (usually no longer existing) lock file and thus over time lots of lock files + accumulated. This has now been fixed. https://github.com/restic/restic/issues/2452 https://github.com/restic/restic/issues/2473 @@ -1679,43 +1878,45 @@ restic users. The changes are ordered by importance. * Bugfix #2738: Don't print progress for `backup --json --quiet` - Unlike the text output, the `--json` output format still printed progress information even in - `--quiet` mode. This has now been fixed by always disabling the progress output in quiet mode. + Unlike the text output, the `--json` output format still printed progress + information even in `--quiet` mode. This has now been fixed by always disabling + the progress output in quiet mode. https://github.com/restic/restic/issues/2738 https://github.com/restic/restic/pull/3264 * Bugfix #3382: Make `check` command honor `RESTIC_CACHE_DIR` environment variable - Previously, the `check` command didn't honor the `RESTIC_CACHE_DIR` environment variable, - which caused problems in certain system/usage configurations. This has now been fixed. + Previously, the `check` command didn't honor the `RESTIC_CACHE_DIR` environment + variable, which caused problems in certain system/usage configurations. This has + now been fixed. https://github.com/restic/restic/issues/3382 https://github.com/restic/restic/pull/3474 * Bugfix #3488: `rebuild-index` failed if an index file was damaged - Previously, the `rebuild-index` command would fail with an error if an index file was damaged - or truncated. This has now been fixed. + Previously, the `rebuild-index` command would fail with an error if an index + file was damaged or truncated. This has now been fixed. - On older restic versions, a (slow) workaround is to use `rebuild-index --read-all-packs` or - to manually delete the damaged index. + On older restic versions, a (slow) workaround is to use `rebuild-index + --read-all-packs` or to manually delete the damaged index. https://github.com/restic/restic/pull/3488 * Bugfix #3518: Make `copy` command honor `--no-lock` for source repository - The `copy` command previously did not respect the `--no-lock` option for the source - repository, causing failures with read-only storage backends. This has now been fixed such - that the option is now respected. + The `copy` command previously did not respect the `--no-lock` option for the + source repository, causing failures with read-only storage backends. This has + now been fixed such that the option is now respected. https://github.com/restic/restic/issues/3518 https://github.com/restic/restic/pull/3589 * Bugfix #3556: Fix hang with Backblaze B2 on SSL certificate authority error - Previously, if a request failed with an SSL unknown certificate authority error, the B2 - backend retried indefinitely and restic would appear to hang. + Previously, if a request failed with an SSL unknown certificate authority error, + the B2 backend retried indefinitely and restic would appear to hang. This has now been fixed and restic instead fails with an error message. @@ -1725,95 +1926,103 @@ restic users. The changes are ordered by importance. * Bugfix #3591: Fix handling of `prune --max-repack-size=0` - Restic ignored the `--max-repack-size` option when passing a value of 0. This has now been - fixed. + Restic ignored the `--max-repack-size` option when passing a value of 0. This + has now been fixed. - As a workaround, `--max-repack-size=1` can be used with older versions of restic. + As a workaround, `--max-repack-size=1` can be used with older versions of + restic. https://github.com/restic/restic/pull/3591 * Bugfix #3601: Fix rclone backend prematurely exiting when receiving SIGINT on Windows - Previously, pressing Ctrl+C in a Windows console where restic was running with rclone as the - backend would cause rclone to exit prematurely due to getting a `SIGINT` signal at the same time - as restic. Restic would then wait for a long time for time with "unexpected EOF" and "rclone - stdio connection already closed" errors. + Previously, pressing Ctrl+C in a Windows console where restic was running with + rclone as the backend would cause rclone to exit prematurely due to getting a + `SIGINT` signal at the same time as restic. Restic would then wait for a long + time for time with "unexpected EOF" and "rclone stdio connection already closed" + errors. - This has now been fixed by restic starting the rclone process detached from the console restic - runs in (similar to starting processes in a new process group on Linux), which enables restic to - gracefully clean up rclone (which now never gets the `SIGINT`). + This has now been fixed by restic starting the rclone process detached from the + console restic runs in (similar to starting processes in a new process group on + Linux), which enables restic to gracefully clean up rclone (which now never gets + the `SIGINT`). https://github.com/restic/restic/issues/3601 https://github.com/restic/restic/pull/3602 * Bugfix #3619: Avoid choosing parent snapshots newer than time of new snapshot - The `backup` command, when a `--parent` was not provided, previously chose the most recent - matching snapshot as the parent snapshot. However, this didn't make sense when the user passed - `--time` to create a new snapshot older than the most recent snapshot. + The `backup` command, when a `--parent` was not provided, previously chose the + most recent matching snapshot as the parent snapshot. However, this didn't make + sense when the user passed `--time` to create a new snapshot older than the most + recent snapshot. - Instead, `backup` now chooses the most recent snapshot which is not newer than the - snapshot-being-created's timestamp, to avoid any time travel. + Instead, `backup` now chooses the most recent snapshot which is not newer than + the snapshot-being-created's timestamp, to avoid any time travel. https://github.com/restic/restic/pull/3619 * Bugfix #3667: The `mount` command now reports symlinks sizes - Symlinks used to have size zero in restic mountpoints, confusing some third-party tools. They - now have a size equal to the byte length of their target path, as required by POSIX. + Symlinks used to have size zero in restic mountpoints, confusing some + third-party tools. They now have a size equal to the byte length of their target + path, as required by POSIX. https://github.com/restic/restic/issues/3667 https://github.com/restic/restic/pull/3668 * Change #3519: Require Go 1.14 or newer - Restic now requires Go 1.14 to build. This allows it to use new standard library features - instead of an external dependency. + Restic now requires Go 1.14 to build. This allows it to use new standard library + features instead of an external dependency. https://github.com/restic/restic/issues/3519 * Change #3641: Ignore parent snapshot for `backup --stdin` - Restic uses a parent snapshot to speed up directory scanning when performing backups, but this - only wasted time and memory when the backup source is stdin (using the `--stdin` option of the - `backup` command), since no directory scanning is performed in this case. + Restic uses a parent snapshot to speed up directory scanning when performing + backups, but this only wasted time and memory when the backup source is stdin + (using the `--stdin` option of the `backup` command), since no directory + scanning is performed in this case. - Snapshots made with `backup --stdin` no longer have a parent snapshot, which allows restic to - skip some startup operations and saves a bit of resources. + Snapshots made with `backup --stdin` no longer have a parent snapshot, which + allows restic to skip some startup operations and saves a bit of resources. - The `--parent` option is still available for `backup --stdin`, but is now ignored. + The `--parent` option is still available for `backup --stdin`, but is now + ignored. https://github.com/restic/restic/issues/3641 https://github.com/restic/restic/pull/3645 * Enhancement #233: Support negative include/exclude patterns - If a pattern starts with an exclamation mark and it matches a file that was previously matched by - a regular pattern, the match is cancelled. Notably, this can be used with `--exclude-file` to - cancel the exclusion of some files. + If a pattern starts with an exclamation mark and it matches a file that was + previously matched by a regular pattern, the match is cancelled. Notably, this + can be used with `--exclude-file` to cancel the exclusion of some files. - It works similarly to `.gitignore`, with the same limitation; Once a directory is excluded, it - is not possible to include files inside the directory. + It works similarly to `.gitignore`, with the same limitation; Once a directory + is excluded, it is not possible to include files inside the directory. Example of use as an exclude pattern for the `backup` command: $HOME/**/* !$HOME/Documents !$HOME/code !$HOME/.emacs.d !$HOME/games # [...] - node_modules *~ *.o *.lo *.pyc # [...] $HOME/code/linux/* !$HOME/code/linux/.git # [...] + node_modules *~ *.o *.lo *.pyc # [...] $HOME/code/linux/* !$HOME/code/linux/.git + # [...] https://github.com/restic/restic/issues/233 https://github.com/restic/restic/pull/2311 * Enhancement #1542: Add `--dry-run`/`-n` option to `backup` command - Testing exclude filters and other configuration options was error prone as wrong filters - could cause files to be uploaded unintentionally. It was also not possible to estimate - beforehand how much data would be uploaded. + Testing exclude filters and other configuration options was error prone as wrong + filters could cause files to be uploaded unintentionally. It was also not + possible to estimate beforehand how much data would be uploaded. - The `backup` command now has a `--dry-run`/`-n` option, which performs all the normal steps of - a backup without actually writing anything to the repository. + The `backup` command now has a `--dry-run`/`-n` option, which performs all the + normal steps of a backup without actually writing anything to the repository. - Passing -vv will log information about files that would be added, allowing for verification of - source and exclusion options before running the real backup. + Passing -vv will log information about files that would be added, allowing for + verification of source and exclusion options before running the real backup. https://github.com/restic/restic/issues/1542 https://github.com/restic/restic/pull/2308 @@ -1822,14 +2031,14 @@ restic users. The changes are ordered by importance. * Enhancement #2202: Add upload checksum for Azure, GS, S3 and Swift backends - Previously only the B2 and partially the Swift backends verified the integrity of uploaded - (encrypted) files. The verification works by informing the backend about the expected hash of - the uploaded file. The backend then verifies the upload and thereby rules out any data - corruption during upload. + Previously only the B2 and partially the Swift backends verified the integrity + of uploaded (encrypted) files. The verification works by informing the backend + about the expected hash of the uploaded file. The backend then verifies the + upload and thereby rules out any data corruption during upload. - We have now added upload checksums for the Azure, GS, S3 and Swift backends, which besides - integrity checking for uploads also means that restic can now be used to store backups in S3 - buckets which have Object Lock enabled. + We have now added upload checksums for the Azure, GS, S3 and Swift backends, + which besides integrity checking for uploads also means that restic can now be + used to store backups in S3 buckets which have Object Lock enabled. https://github.com/restic/restic/issues/2202 https://github.com/restic/restic/issues/2700 @@ -1838,65 +2047,68 @@ restic users. The changes are ordered by importance. * Enhancement #2388: Add warning for S3 if partial credentials are provided - Previously restic did not notify about incomplete credentials when using the S3 backend, - instead just reporting access denied. + Previously restic did not notify about incomplete credentials when using the S3 + backend, instead just reporting access denied. - Restic now checks that both the AWS key ID and secret environment variables are set before - connecting to the remote server, and reports an error if not. + Restic now checks that both the AWS key ID and secret environment variables are + set before connecting to the remote server, and reports an error if not. https://github.com/restic/restic/issues/2388 https://github.com/restic/restic/pull/3532 * Enhancement #2508: Support JSON output and quiet mode for the `diff` command - The `diff` command now supports outputting machine-readable output in JSON format. To enable - this, pass the `--json` option to the command. To only print the summary and suppress detailed - output, pass the `--quiet` option. + The `diff` command now supports outputting machine-readable output in JSON + format. To enable this, pass the `--json` option to the command. To only print + the summary and suppress detailed output, pass the `--quiet` option. https://github.com/restic/restic/issues/2508 https://github.com/restic/restic/pull/3592 * Enhancement #2594: Speed up the `restore --verify` command - The `--verify` option lets the `restore` command verify the file content after it has restored - a snapshot. The performance of this operation has now been improved by up to a factor of two. + The `--verify` option lets the `restore` command verify the file content after + it has restored a snapshot. The performance of this operation has now been + improved by up to a factor of two. https://github.com/restic/restic/pull/2594 * Enhancement #2656: Add flag to disable TLS verification for self-signed certificates - There is now an `--insecure-tls` global option in restic, which disables TLS verification for - self-signed certificates in order to support some development workflows. + There is now an `--insecure-tls` global option in restic, which disables TLS + verification for self-signed certificates in order to support some development + workflows. https://github.com/restic/restic/issues/2656 https://github.com/restic/restic/pull/2657 * Enhancement #2816: The `backup` command no longer updates file access times on Linux - When reading files during backup, restic used to cause the operating system to update the - files' access times. Note that this did not apply to filesystems with disabled file access - times. + When reading files during backup, restic used to cause the operating system to + update the files' access times. Note that this did not apply to filesystems with + disabled file access times. - Restic now instructs the operating system not to update the file access time, if the user - running restic is the file owner or has root permissions. + Restic now instructs the operating system not to update the file access time, if + the user running restic is the file owner or has root permissions. https://github.com/restic/restic/pull/2816 * Enhancement #2880: Make `recover` collect only unreferenced trees - Previously, the `recover` command used to generate a snapshot containing *all* root trees, - even those which were already referenced by a snapshot. + Previously, the `recover` command used to generate a snapshot containing *all* + root trees, even those which were already referenced by a snapshot. - This has been improved such that it now only processes trees not already referenced by any - snapshot. + This has been improved such that it now only processes trees not already + referenced by any snapshot. https://github.com/restic/restic/pull/2880 * Enhancement #3003: Atomic uploads for the SFTP backend - The SFTP backend did not upload files atomically. An interrupted upload could leave an - incomplete file behind which could prevent restic from accessing the repository. This has now - been fixed and uploads in the SFTP backend are done atomically. + The SFTP backend did not upload files atomically. An interrupted upload could + leave an incomplete file behind which could prevent restic from accessing the + repository. This has now been fixed and uploads in the SFTP backend are done + atomically. https://github.com/restic/restic/issues/3003 https://github.com/restic/restic/pull/3524 @@ -1910,25 +2122,27 @@ restic users. The changes are ordered by importance. * Enhancement #3429: Verify that new or modified keys are stored correctly - When adding a new key or changing the password of a key, restic used to just create the new key (and - remove the old one, when changing the password). There was no verification that the new key was - stored correctly and works properly. As the repository cannot be decrypted without a valid key - file, this could in rare cases cause the repository to become inaccessible. + When adding a new key or changing the password of a key, restic used to just + create the new key (and remove the old one, when changing the password). There + was no verification that the new key was stored correctly and works properly. As + the repository cannot be decrypted without a valid key file, this could in rare + cases cause the repository to become inaccessible. - Restic now checks that new key files actually work before continuing. This can protect against - some (rare) cases of hardware or storage problems. + Restic now checks that new key files actually work before continuing. This can + protect against some (rare) cases of hardware or storage problems. https://github.com/restic/restic/pull/3429 * Enhancement #3436: Improve local backend's resilience to (system) crashes - Restic now ensures that files stored using the `local` backend are created atomically (that - is, files are either stored completely or not at all). This ensures that no incomplete files are - left behind even if restic is terminated while writing a file. + Restic now ensures that files stored using the `local` backend are created + atomically (that is, files are either stored completely or not at all). This + ensures that no incomplete files are left behind even if restic is terminated + while writing a file. - In addition, restic now tries to ensure that the directory in the repository which contains a - newly uploaded file is also written to disk. This can prevent missing files if the system - crashes or the disk is not properly unmounted. + In addition, restic now tries to ensure that the directory in the repository + which contains a newly uploaded file is also written to disk. This can prevent + missing files if the system crashes or the disk is not properly unmounted. https://github.com/restic/restic/pull/3436 @@ -1936,54 +2150,56 @@ restic users. The changes are ordered by importance. Restic used to silently ignore the `--no-lock` option of the `forget` command. - It now skips creation of lock file in case both `--dry-run` and `--no-lock` are specified. If - `--no-lock` option is specified without `--dry-run`, restic prints a warning message to - stderr. + It now skips creation of lock file in case both `--dry-run` and `--no-lock` are + specified. If `--no-lock` option is specified without `--dry-run`, restic prints + a warning message to stderr. https://github.com/restic/restic/issues/3464 https://github.com/restic/restic/pull/3623 * Enhancement #3490: Support random subset by size in `check --read-data-subset` - The `--read-data-subset` option of the `check` command now supports a third way of specifying - the subset to check, namely `nS` where `n` is a size in bytes with suffix `S` as k/K, m/M, g/G or - t/T. + The `--read-data-subset` option of the `check` command now supports a third way + of specifying the subset to check, namely `nS` where `n` is a size in bytes with + suffix `S` as k/K, m/M, g/G or t/T. https://github.com/restic/restic/issues/3490 https://github.com/restic/restic/pull/3548 * Enhancement #3508: Cache blobs read by the `dump` command - When dumping a file using the `dump` command, restic did not cache blobs in any way, so even - consecutive runs of the same blob were loaded from the repository again and again, slowing down - the dump. + When dumping a file using the `dump` command, restic did not cache blobs in any + way, so even consecutive runs of the same blob were loaded from the repository + again and again, slowing down the dump. - Now, the caching mechanism already used by the `fuse` command is also used by the `dump` - command. This makes dumping much faster, especially for sparse files. + Now, the caching mechanism already used by the `fuse` command is also used by + the `dump` command. This makes dumping much faster, especially for sparse files. https://github.com/restic/restic/pull/3508 * Enhancement #3511: Support configurable timeout for the rclone backend - A slow rclone backend could cause restic to time out while waiting for the repository to open. - Restic now offers an `-o rclone.timeout` option to make this timeout configurable. + A slow rclone backend could cause restic to time out while waiting for the + repository to open. Restic now offers an `-o rclone.timeout` option to make this + timeout configurable. https://github.com/restic/restic/issues/3511 https://github.com/restic/restic/pull/3514 * Enhancement #3541: Improve handling of temporary B2 delete errors - Deleting files on B2 could sometimes fail temporarily, which required restic to retry the - delete operation. In some cases the file was deleted nevertheless, causing the retries and - ultimately the restic command to fail. This has now been fixed. + Deleting files on B2 could sometimes fail temporarily, which required restic to + retry the delete operation. In some cases the file was deleted nevertheless, + causing the retries and ultimately the restic command to fail. This has now been + fixed. https://github.com/restic/restic/issues/3541 https://github.com/restic/restic/pull/3544 * Enhancement #3542: Add file mode in symbolic notation to `ls --json` - The `ls --json` command now provides the file mode in symbolic notation (using the - `permissions` key), aligned with `find --json`. + The `ls --json` command now provides the file mode in symbolic notation (using + the `permissions` key), aligned with `find --json`. https://github.com/restic/restic/issues/3542 https://github.com/restic/restic/pull/3573 @@ -1991,11 +2207,12 @@ restic users. The changes are ordered by importance. * Enhancement #3593: Improve `copy` performance by parallelizing IO - Restic copy previously only used a single thread for copying blobs between repositories, - which resulted in limited performance when copying small blobs to/from a high latency backend - (i.e. any remote backend, especially b2). + Restic copy previously only used a single thread for copying blobs between + repositories, which resulted in limited performance when copying small blobs + to/from a high latency backend (i.e. any remote backend, especially b2). - Copying will now use 8 parallel threads to increase the throughput of the copy operation. + Copying will now use 8 parallel threads to increase the throughput of the copy + operation. https://github.com/restic/restic/pull/3593 @@ -2033,9 +2250,9 @@ restic users. The changes are ordered by importance. * Bugfix #2742: Improve error handling for rclone and REST backend over HTTP2 - When retrieving data from the rclone / REST backend while also using HTTP2 restic did not detect - when no data was returned at all. This could cause for example the `check` command to report the - following error: + When retrieving data from the rclone / REST backend while also using HTTP2 + restic did not detect when no data was returned at all. This could cause for + example the `check` command to report the following error: Pack ID does not match, want [...], got e3b0c442 @@ -2047,98 +2264,105 @@ restic users. The changes are ordered by importance. * Bugfix #3111: Fix terminal output redirection for PowerShell - When redirecting the output of restic using PowerShell on Windows, the output contained - terminal escape characters. This has been fixed by properly detecting the terminal type. + When redirecting the output of restic using PowerShell on Windows, the output + contained terminal escape characters. This has been fixed by properly detecting + the terminal type. - In addition, the mintty terminal now shows progress output for the backup command. + In addition, the mintty terminal now shows progress output for the backup + command. https://github.com/restic/restic/issues/3111 https://github.com/restic/restic/pull/3325 * Bugfix #3184: `backup --quiet` no longer prints status information - A regression in the latest restic version caused the output of `backup --quiet` to contain - large amounts of backup progress information when run using an interactive terminal. This is - fixed now. + A regression in the latest restic version caused the output of `backup --quiet` + to contain large amounts of backup progress information when run using an + interactive terminal. This is fixed now. - A workaround for this bug is to run restic as follows: `restic backup --quiet [..] | cat -`. + A workaround for this bug is to run restic as follows: `restic backup --quiet + [..] | cat -`. https://github.com/restic/restic/issues/3184 https://github.com/restic/restic/pull/3186 * Bugfix #3214: Treat an empty password as a fatal error for repository init - When attempting to initialize a new repository, if an empty password was supplied, the - repository would be created but the init command would return an error with a stack trace. Now, - if an empty password is provided, it is treated as a fatal error, and no repository is created. + When attempting to initialize a new repository, if an empty password was + supplied, the repository would be created but the init command would return an + error with a stack trace. Now, if an empty password is provided, it is treated + as a fatal error, and no repository is created. https://github.com/restic/restic/issues/3214 https://github.com/restic/restic/pull/3283 * Bugfix #3267: `copy` failed to copy snapshots in rare cases - The `copy` command could in rare cases fail with the error message `SaveTree(...) returned - unexpected id ...`. This has been fixed. + The `copy` command could in rare cases fail with the error message + `SaveTree(...) returned unexpected id ...`. This has been fixed. - On Linux/BSDs, the error could be caused by backing up symlinks with non-UTF-8 target paths. - Note that, due to limitations in the repository format, these are not stored properly and - should be avoided if possible. + On Linux/BSDs, the error could be caused by backing up symlinks with non-UTF-8 + target paths. Note that, due to limitations in the repository format, these are + not stored properly and should be avoided if possible. https://github.com/restic/restic/issues/3267 https://github.com/restic/restic/pull/3310 * Bugfix #3296: Fix crash of `check --read-data-subset=x%` run for an empty repository - The command `restic check --read-data-subset=x%` crashed when run for an empty repository. - This has been fixed. + The command `restic check --read-data-subset=x%` crashed when run for an empty + repository. This has been fixed. https://github.com/restic/restic/issues/3296 https://github.com/restic/restic/pull/3309 * Bugfix #3302: Fix `fdopendir: not a directory` error for local backend - The `check`, `list packs`, `prune` and `rebuild-index` commands failed for the local backend - when the `data` folder in the repository contained files. This has been fixed. + The `check`, `list packs`, `prune` and `rebuild-index` commands failed for the + local backend when the `data` folder in the repository contained files. This has + been fixed. https://github.com/restic/restic/issues/3302 https://github.com/restic/restic/pull/3308 * Bugfix #3305: Fix possibly missing backup summary of JSON output in case of error - When using `--json` output it happened from time to time that the summary output was missing in - case an error occurred. This has been fixed. + When using `--json` output it happened from time to time that the summary output + was missing in case an error occurred. This has been fixed. https://github.com/restic/restic/pull/3305 * Bugfix #3334: Print `created new cache` message only on a terminal - The message `created new cache` was printed even when the output wasn't a terminal. That broke - piping `restic dump` output to tar or zip if cache directory didn't exist. The message is now - only printed on a terminal. + The message `created new cache` was printed even when the output wasn't a + terminal. That broke piping `restic dump` output to tar or zip if cache + directory didn't exist. The message is now only printed on a terminal. https://github.com/restic/restic/issues/3334 https://github.com/restic/restic/pull/3343 * Bugfix #3380: Fix crash of `backup --exclude='**'` - The exclude filter `**`, which excludes all files, caused restic to crash. This has been - corrected. + The exclude filter `**`, which excludes all files, caused restic to crash. This + has been corrected. https://github.com/restic/restic/issues/3380 https://github.com/restic/restic/pull/3393 * Bugfix #3439: Correctly handle download errors during `restore` - Due to a regression in restic 0.12.0, the `restore` command in some cases did not retry download - errors and only printed a warning. This has been fixed by retrying incomplete data downloads. + Due to a regression in restic 0.12.0, the `restore` command in some cases did + not retry download errors and only printed a warning. This has been fixed by + retrying incomplete data downloads. https://github.com/restic/restic/issues/3439 https://github.com/restic/restic/pull/3449 * Change #3247: Empty files now have size of 0 in `ls --json` output - The `ls --json` command used to omit the sizes of empty files in its output. It now reports a size - of zero explicitly for regular files, while omitting the size field for all other types. + The `ls --json` command used to omit the sizes of empty files in its output. It + now reports a size of zero explicitly for regular files, while omitting the size + field for all other types. https://github.com/restic/restic/issues/3247 https://github.com/restic/restic/pull/3257 @@ -2152,9 +2376,9 @@ restic users. The changes are ordered by importance. * Enhancement #3167: Allow specifying limit of `snapshots` list - The `--last` option allowed limiting the output of the `snapshots` command to the latest - snapshot for each host. The new `--latest n` option allows limiting the output to the latest `n` - snapshots. + The `--last` option allowed limiting the output of the `snapshots` command to + the latest snapshot for each host. The new `--latest n` option allows limiting + the output to the latest `n` snapshots. This change deprecates the option `--last` in favour of `--latest 1`. @@ -2162,13 +2386,15 @@ restic users. The changes are ordered by importance. * Enhancement #3293: Add `--repository-file2` option to `init` and `copy` command - The `init` and `copy` command can now be used with the `--repository-file2` option or the - `$RESTIC_REPOSITORY_FILE2` environment variable. These to options are in addition to the - `--repo2` flag and allow you to read the destination repository from a file. + The `init` and `copy` command can now be used with the `--repository-file2` + option or the `$RESTIC_REPOSITORY_FILE2` environment variable. These to options + are in addition to the `--repo2` flag and allow you to read the destination + repository from a file. - Using both `--repository-file` and `--repo2` options resulted in an error for the `copy` or - `init` command. The handling of this combination of options has been fixed. A workaround for - this issue is to only use `--repo` or `-r` and `--repo2` for `init` or `copy`. + Using both `--repository-file` and `--repo2` options resulted in an error for + the `copy` or `init` command. The handling of this combination of options has + been fixed. A workaround for this issue is to only use `--repo` or `-r` and + `--repo2` for `init` or `copy`. https://github.com/restic/restic/issues/3293 https://github.com/restic/restic/pull/3294 @@ -2181,9 +2407,9 @@ restic users. The changes are ordered by importance. * Enhancement #3336: SFTP backend now checks for disk space - Backing up over SFTP previously spewed multiple generic "failure" messages when the remote - disk was full. It now checks for disk space before writing a file and fails immediately with a "no - space left on device" message. + Backing up over SFTP previously spewed multiple generic "failure" messages when + the remote disk was full. It now checks for disk space before writing a file and + fails immediately with a "no space left on device" message. https://github.com/restic/restic/issues/3336 https://github.com/restic/restic/pull/3345 @@ -2197,15 +2423,17 @@ restic users. The changes are ordered by importance. * Enhancement #3414: Add `--keep-within-hourly` option to restic forget - The `forget` command allowed keeping a given number of hourly backups or to keep all backups - within a given interval, but it was not possible to specify keeping hourly backups within a - given interval. + The `forget` command allowed keeping a given number of hourly backups or to keep + all backups within a given interval, but it was not possible to specify keeping + hourly backups within a given interval. - The new `--keep-within-hourly` option now offers this functionality. Similar options for - daily/weekly/monthly/yearly are also implemented, the new options are: + The new `--keep-within-hourly` option now offers this functionality. Similar + options for daily/weekly/monthly/yearly are also implemented, the new options + are: - --keep-within-hourly <1y2m3d4h> --keep-within-daily <1y2m3d4h> --keep-within-weekly - <1y2m3d4h> --keep-within-monthly <1y2m3d4h> --keep-within-yearly <1y2m3d4h> + --keep-within-hourly <1y2m3d4h> --keep-within-daily <1y2m3d4h> + --keep-within-weekly <1y2m3d4h> --keep-within-monthly <1y2m3d4h> + --keep-within-yearly <1y2m3d4h> https://github.com/restic/restic/issues/3414 https://github.com/restic/restic/pull/3416 @@ -2213,30 +2441,32 @@ restic users. The changes are ordered by importance. * Enhancement #3426: Optimize read performance of mount command - Reading large files in a mounted repository may be up to five times faster. This improvement - primarily applies to repositories stored at a backend that can be accessed with low latency, - like e.g. the local backend. + Reading large files in a mounted repository may be up to five times faster. This + improvement primarily applies to repositories stored at a backend that can be + accessed with low latency, like e.g. the local backend. https://github.com/restic/restic/pull/3426 * Enhancement #3427: `find --pack` fallback to index if data file is missing - When investigating a repository with missing data files, it might be useful to determine - affected snapshots before running `rebuild-index`. Previously, `find --pack pack-id` - returned no data as it required accessing the data file. Now, if the necessary data is still - available in the repository index, it gets retrieved from there. + When investigating a repository with missing data files, it might be useful to + determine affected snapshots before running `rebuild-index`. Previously, `find + --pack pack-id` returned no data as it required accessing the data file. Now, if + the necessary data is still available in the repository index, it gets retrieved + from there. - The command now also supports looking up multiple pack files in a single `find` run. + The command now also supports looking up multiple pack files in a single `find` + run. https://github.com/restic/restic/pull/3427 https://forum.restic.net/t/missing-packs-not-found/2600 * Enhancement #3456: Support filtering and specifying untagged snapshots - It was previously not possible to specify an empty tag with the `--tag` and `--keep-tag` - options. This has now been fixed, such that `--tag ''` and `--keep-tag ''` now matches - snapshots without tags. This allows e.g. the `snapshots` and `forget` commands to only - operate on untagged snapshots. + It was previously not possible to specify an empty tag with the `--tag` and + `--keep-tag` options. This has now been fixed, such that `--tag ''` and + `--keep-tag ''` now matches snapshots without tags. This allows e.g. the + `snapshots` and `forget` commands to only operate on untagged snapshots. https://github.com/restic/restic/issues/3456 https://github.com/restic/restic/pull/3457 @@ -2288,28 +2518,28 @@ restic users. The changes are ordered by importance. * Bugfix #1681: Make `mount` not create missing mount point directory - When specifying a non-existent directory as mount point for the `mount` command, restic used - to create the specified directory automatically. + When specifying a non-existent directory as mount point for the `mount` command, + restic used to create the specified directory automatically. - This has now changed such that restic instead gives an error when the specified directory for - the mount point does not exist. + This has now changed such that restic instead gives an error when the specified + directory for the mount point does not exist. https://github.com/restic/restic/issues/1681 https://github.com/restic/restic/pull/3008 * Bugfix #1800: Ignore `no data available` filesystem error during backup - Restic was unable to backup files on some filesystems, for example certain configurations of - CIFS on Linux which return a `no data available` error when reading extended attributes. These - errors are now ignored. + Restic was unable to backup files on some filesystems, for example certain + configurations of CIFS on Linux which return a `no data available` error when + reading extended attributes. These errors are now ignored. https://github.com/restic/restic/issues/1800 https://github.com/restic/restic/pull/3034 * Bugfix #2563: Report the correct owner of directories in FUSE mounts - Restic 0.10.0 changed the FUSE mount to always report the current user as the owner of - directories within the FUSE mount, which is incorrect. + Restic 0.10.0 changed the FUSE mount to always report the current user as the + owner of directories within the FUSE mount, which is incorrect. This is now changed back to reporting the correct owner of a directory. @@ -2318,30 +2548,31 @@ restic users. The changes are ordered by importance. * Bugfix #2688: Make `backup` and `tag` commands separate tags by comma - Running `restic backup --tag foo,bar` previously created snapshots with one single tag - containing a comma (`foo,bar`) instead of two tags (`foo`, `bar`). + Running `restic backup --tag foo,bar` previously created snapshots with one + single tag containing a comma (`foo,bar`) instead of two tags (`foo`, `bar`). - Similarly, the `tag` command's `--set`, `--add` and `--remove` options would treat - `foo,bar` as one tag instead of two tags. This was inconsistent with other commands and often - unexpected when one intended `foo,bar` to mean two tags. + Similarly, the `tag` command's `--set`, `--add` and `--remove` options would + treat `foo,bar` as one tag instead of two tags. This was inconsistent with other + commands and often unexpected when one intended `foo,bar` to mean two tags. - To be consistent in all commands, restic now interprets `foo,bar` to mean two separate tags - (`foo` and `bar`) instead of one tag (`foo,bar`) everywhere, including in the `backup` and - `tag` commands. + To be consistent in all commands, restic now interprets `foo,bar` to mean two + separate tags (`foo` and `bar`) instead of one tag (`foo,bar`) everywhere, + including in the `backup` and `tag` commands. - NOTE: This change might result in unexpected behavior in cases where you use the `forget` - command and filter on tags like `foo,bar`. Snapshots previously backed up with `--tag - foo,bar` will still not match that filter, but snapshots saved from now on will match that - filter. + NOTE: This change might result in unexpected behavior in cases where you use the + `forget` command and filter on tags like `foo,bar`. Snapshots previously backed + up with `--tag foo,bar` will still not match that filter, but snapshots saved + from now on will match that filter. - To replace `foo,bar` tags with `foo` and `bar` tags in old snapshots, you can first generate a - list of the relevant snapshots using a command like: + To replace `foo,bar` tags with `foo` and `bar` tags in old snapshots, you can + first generate a list of the relevant snapshots using a command like: - Restic snapshots --json --quiet | jq '.[] | select(contains({tags: ["foo,bar"]})) | .id' + Restic snapshots --json --quiet | jq '.[] | select(contains({tags: + ["foo,bar"]})) | .id' - And then use `restic tag --set foo --set bar snapshotID [...]` to set the new tags. Please adjust - the commands to include real tag names and any additional tags, as well as the list of snapshots - to process. + And then use `restic tag --set foo --set bar snapshotID [...]` to set the new + tags. Please adjust the commands to include real tag names and any additional + tags, as well as the list of snapshots to process. https://github.com/restic/restic/issues/2688 https://github.com/restic/restic/pull/2690 @@ -2355,14 +2586,14 @@ restic users. The changes are ordered by importance. * Bugfix #3014: Fix sporadic stream reset between rclone and restic - Sometimes when using restic with the `rclone` backend, an error message similar to the - following would be printed: + Sometimes when using restic with the `rclone` backend, an error message similar + to the following would be printed: Didn't finish writing GET request (wrote 0/xxx): http2: stream closed - It was found that this was caused by restic closing the connection to rclone to soon when - downloading data. A workaround has been added which waits for the end of the download before - closing the connection. + It was found that this was caused by restic closing the connection to rclone to + soon when downloading data. A workaround has been added which waits for the end + of the download before closing the connection. https://github.com/rclone/rclone/issues/2598 https://github.com/restic/restic/pull/3014 @@ -2380,125 +2611,130 @@ restic users. The changes are ordered by importance. * Bugfix #3100: Do not require gs bucket permissions when running `init` - Restic used to require bucket level permissions for the `gs` backend in order to initialize a - restic repository. + Restic used to require bucket level permissions for the `gs` backend in order to + initialize a restic repository. - It now allows a `gs` service account to initialize a repository if the bucket does exist and the - service account has permissions to write/read to that bucket. + It now allows a `gs` service account to initialize a repository if the bucket + does exist and the service account has permissions to write/read to that bucket. https://github.com/restic/restic/issues/3100 * Bugfix #3111: Correctly detect output redirection for `backup` command on Windows - On Windows, since restic 0.10.0 the `backup` command did not properly detect when the output - was redirected to a file. This caused restic to output terminal control characters. This has - been fixed by correcting the terminal detection. + On Windows, since restic 0.10.0 the `backup` command did not properly detect + when the output was redirected to a file. This caused restic to output terminal + control characters. This has been fixed by correcting the terminal detection. https://github.com/restic/restic/issues/3111 https://github.com/restic/restic/pull/3150 * Bugfix #3151: Don't create invalid snapshots when `backup` is interrupted - When canceling a backup run at a certain moment it was possible that restic created a snapshot - with an invalid "null" tree. This caused `check` and other operations to fail. The `backup` - command now properly handles interruptions and never saves a snapshot when interrupted. + When canceling a backup run at a certain moment it was possible that restic + created a snapshot with an invalid "null" tree. This caused `check` and other + operations to fail. The `backup` command now properly handles interruptions and + never saves a snapshot when interrupted. https://github.com/restic/restic/issues/3151 https://github.com/restic/restic/pull/3164 * Bugfix #3152: Do not hang until foregrounded when completed in background - On Linux, when running in the background restic failed to stop the terminal output of the - `backup` command after it had completed. This caused restic to hang until moved to the - foreground. This has now been fixed. + On Linux, when running in the background restic failed to stop the terminal + output of the `backup` command after it had completed. This caused restic to + hang until moved to the foreground. This has now been fixed. https://github.com/restic/restic/pull/3152 https://forum.restic.net/t/restic-alpine-container-cron-hangs-epoll-pwait/3334 * Bugfix #3166: Improve error handling in the `restore` command - The `restore` command used to not print errors while downloading file contents from the - repository. It also incorrectly exited with a zero error code even when there were errors - during the restore process. This has all been fixed and `restore` now returns with a non-zero - exit code when there's an error. + The `restore` command used to not print errors while downloading file contents + from the repository. It also incorrectly exited with a zero error code even when + there were errors during the restore process. This has all been fixed and + `restore` now returns with a non-zero exit code when there's an error. https://github.com/restic/restic/issues/3166 https://github.com/restic/restic/pull/3207 * Bugfix #3232: Correct statistics for overlapping targets - A user reported that restic's statistics and progress information during backup was not - correctly calculated when the backup targets (files/dirs to save) overlap. For example, - consider a directory `foo` which contains (among others) a file `foo/bar`. When `restic - backup foo foo/bar` was run, restic counted the size of the file `foo/bar` twice, so the - completeness percentage as well as the number of files was wrong. This is now corrected. + A user reported that restic's statistics and progress information during backup + was not correctly calculated when the backup targets (files/dirs to save) + overlap. For example, consider a directory `foo` which contains (among others) a + file `foo/bar`. When `restic backup foo foo/bar` was run, restic counted the + size of the file `foo/bar` twice, so the completeness percentage as well as the + number of files was wrong. This is now corrected. https://github.com/restic/restic/issues/3232 https://github.com/restic/restic/pull/3243 * Bugfix #3249: Improve error handling in `gs` backend - The `gs` backend did not notice when the last step of completing a file upload failed. Under rare - circumstances, this could cause missing files in the backup repository. This has now been - fixed. + The `gs` backend did not notice when the last step of completing a file upload + failed. Under rare circumstances, this could cause missing files in the backup + repository. This has now been fixed. https://github.com/restic/restic/pull/3249 * Change #3095: Deleting files on Google Drive now moves them to the trash - When deleting files on Google Drive via the `rclone` backend, restic used to bypass the trash - folder required that one used the `-o rclone.args` option to enable usage of the trash folder. - This ensured that deleted files in Google Drive were not kept indefinitely in the trash folder. - However, since Google Drive's trash retention policy changed to deleting trashed files after - 30 days, this is no longer needed. + When deleting files on Google Drive via the `rclone` backend, restic used to + bypass the trash folder required that one used the `-o rclone.args` option to + enable usage of the trash folder. This ensured that deleted files in Google + Drive were not kept indefinitely in the trash folder. However, since Google + Drive's trash retention policy changed to deleting trashed files after 30 days, + this is no longer needed. - Restic now leaves it up to rclone and its configuration to use or not use the trash folder when - deleting files. The default is to use the trash folder, as of rclone 1.53.2. To re-enable the - restic 0.11 behavior, set the `RCLONE_DRIVE_USE_TRASH` environment variable or change the - rclone configuration. See the rclone documentation for more details. + Restic now leaves it up to rclone and its configuration to use or not use the + trash folder when deleting files. The default is to use the trash folder, as of + rclone 1.53.2. To re-enable the restic 0.11 behavior, set the + `RCLONE_DRIVE_USE_TRASH` environment variable or change the rclone + configuration. See the rclone documentation for more details. https://github.com/restic/restic/issues/3095 https://github.com/restic/restic/pull/3102 * Enhancement #909: Back up mountpoints as empty directories - When the `--one-file-system` option is specified to `restic backup`, it ignores all file - systems mounted below one of the target directories. This means that when a snapshot is - restored, users needed to manually recreate the mountpoint directories. + When the `--one-file-system` option is specified to `restic backup`, it ignores + all file systems mounted below one of the target directories. This means that + when a snapshot is restored, users needed to manually recreate the mountpoint + directories. - Restic now backs up mountpoints as empty directories and therefore implements the same - approach as `tar`. + Restic now backs up mountpoints as empty directories and therefore implements + the same approach as `tar`. https://github.com/restic/restic/issues/909 https://github.com/restic/restic/pull/3119 * Enhancement #2186: Allow specifying percentage in `check --read-data-subset` - We've enhanced the `check` command's `--read-data-subset` option to also accept a - percentage (e.g. `2.5%` or `10%`). This will check the given percentage of pack files (which - are randomly selected on each run). + We've enhanced the `check` command's `--read-data-subset` option to also accept + a percentage (e.g. `2.5%` or `10%`). This will check the given percentage of + pack files (which are randomly selected on each run). https://github.com/restic/restic/issues/2186 https://github.com/restic/restic/pull/3038 * Enhancement #2433: Make the `dump` command support `zip` format - Previously, restic could dump the contents of a whole folder structure only in the `tar` - format. The `dump` command now has a new flag to change output format to `zip`. Just pass - `--archive zip` as an option to `restic dump`. + Previously, restic could dump the contents of a whole folder structure only in + the `tar` format. The `dump` command now has a new flag to change output format + to `zip`. Just pass `--archive zip` as an option to `restic dump`. https://github.com/restic/restic/pull/2433 https://github.com/restic/restic/pull/3081 * Enhancement #2453: Report permanent/fatal backend errors earlier - When encountering errors in reading from or writing to storage backends, restic retries the - failing operation up to nine times (for a total of ten attempts). It used to retry all backend - operations, but now detects some permanent error conditions so that it can report fatal errors - earlier. + When encountering errors in reading from or writing to storage backends, restic + retries the failing operation up to nine times (for a total of ten attempts). It + used to retry all backend operations, but now detects some permanent error + conditions so that it can report fatal errors earlier. - Permanent failures include local disks being full, SSH connections dropping and permission - errors. + Permanent failures include local disks being full, SSH connections dropping and + permission errors. https://github.com/restic/restic/issues/2453 https://github.com/restic/restic/issues/3180 @@ -2507,23 +2743,26 @@ restic users. The changes are ordered by importance. * Enhancement #2495: Add option to let `backup` trust mtime without checking ctime - The `backup` command used to require that both `ctime` and `mtime` of a file matched with a - previously backed up version to determine that the file was unchanged. In other words, if - either `ctime` or `mtime` of the file had changed, it would be considered changed and restic - would read the file's content again to back up the relevant (changed) parts of it. + The `backup` command used to require that both `ctime` and `mtime` of a file + matched with a previously backed up version to determine that the file was + unchanged. In other words, if either `ctime` or `mtime` of the file had changed, + it would be considered changed and restic would read the file's content again to + back up the relevant (changed) parts of it. - The new option `--ignore-ctime` makes restic look at `mtime` only, such that `ctime` changes - for a file does not cause restic to read the file's contents again. + The new option `--ignore-ctime` makes restic look at `mtime` only, such that + `ctime` changes for a file does not cause restic to read the file's contents + again. - The check for both `ctime` and `mtime` was introduced in restic 0.9.6 to make backups more - reliable in the face of programs that reset `mtime` (some Unix archivers do that), but it turned - out to often be expensive because it made restic read file contents even if only the metadata - (owner, permissions) of a file had changed. The new `--ignore-ctime` option lets the user - restore the 0.9.5 behavior when needed. The existing `--ignore-inode` option already turned + The check for both `ctime` and `mtime` was introduced in restic 0.9.6 to make + backups more reliable in the face of programs that reset `mtime` (some Unix + archivers do that), but it turned out to often be expensive because it made + restic read file contents even if only the metadata (owner, permissions) of a + file had changed. The new `--ignore-ctime` option lets the user restore the + 0.9.5 behavior when needed. The existing `--ignore-inode` option already turned off this behavior, but also removed a different check. - Please note that changes in files' metadata are still recorded, regardless of the command line - options provided to the backup command. + Please note that changes in files' metadata are still recorded, regardless of + the command line options provided to the backup command. https://github.com/restic/restic/issues/2495 https://github.com/restic/restic/issues/2558 @@ -2532,20 +2771,21 @@ restic users. The changes are ordered by importance. * Enhancement #2528: Add Alibaba/Aliyun OSS support in the `s3` backend - A new extended option `s3.bucket-lookup` has been added to support Alibaba/Aliyun OSS in the - `s3` backend. The option can be set to one of the following values: + A new extended option `s3.bucket-lookup` has been added to support + Alibaba/Aliyun OSS in the `s3` backend. The option can be set to one of the + following values: - - `auto` - Existing behaviour - `dns` - Use DNS style bucket access - `path` - Use path style - bucket access + - `auto` - Existing behaviour - `dns` - Use DNS style bucket access - `path` - + Use path style bucket access - To make the `s3` backend work with Alibaba/Aliyun OSS you must set `s3.bucket-lookup` to `dns` - and set the `s3.region` parameter. For example: + To make the `s3` backend work with Alibaba/Aliyun OSS you must set + `s3.bucket-lookup` to `dns` and set the `s3.region` parameter. For example: Restic -o s3.bucket-lookup=dns -o s3.region=oss-eu-west-1 -r s3:https://oss-eu-west-1.aliyuncs.com/bucketname init - Note that `s3.region` must be set, otherwise the MinIO SDK tries to look it up and it seems that - Alibaba doesn't support that properly. + Note that `s3.region` must be set, otherwise the MinIO SDK tries to look it up + and it seems that Alibaba doesn't support that properly. https://github.com/restic/restic/issues/2528 https://github.com/restic/restic/pull/2535 @@ -2554,14 +2794,14 @@ restic users. The changes are ordered by importance. The `backup`, `check` and `prune` commands never printed any progress reports on non-interactive terminals. This behavior is now configurable using the - `RESTIC_PROGRESS_FPS` environment variable. Use for example a value of `1` for an update - every second, or `0.01666` for an update every minute. + `RESTIC_PROGRESS_FPS` environment variable. Use for example a value of `1` for + an update every second, or `0.01666` for an update every minute. - The `backup` command now also prints the current progress when restic receives a `SIGUSR1` - signal. + The `backup` command now also prints the current progress when restic receives a + `SIGUSR1` signal. - Setting the `RESTIC_PROGRESS_FPS` environment variable or sending a `SIGUSR1` signal - prints a status report even when `--quiet` was specified. + Setting the `RESTIC_PROGRESS_FPS` environment variable or sending a `SIGUSR1` + signal prints a status report even when `--quiet` was specified. https://github.com/restic/restic/issues/2706 https://github.com/restic/restic/issues/3194 @@ -2569,21 +2809,22 @@ restic users. The changes are ordered by importance. * Enhancement #2718: Improve `prune` performance and make it more customizable - The `prune` command is now much faster. This is especially the case for remote repositories or - repositories with not much data to remove. Also the memory usage of the `prune` command is now - reduced. + The `prune` command is now much faster. This is especially the case for remote + repositories or repositories with not much data to remove. Also the memory usage + of the `prune` command is now reduced. - Restic used to rebuild the index from scratch after pruning. This could lead to missing packs in - the index in some cases for eventually consistent backends such as e.g. AWS S3. This behavior is - now changed and the index rebuilding uses the information already known by `prune`. + Restic used to rebuild the index from scratch after pruning. This could lead to + missing packs in the index in some cases for eventually consistent backends such + as e.g. AWS S3. This behavior is now changed and the index rebuilding uses the + information already known by `prune`. - By default, the `prune` command no longer removes all unused data. This behavior can be - fine-tuned by new options, like the acceptable amount of unused space or the maximum size of - data to reorganize. For more details, please see + By default, the `prune` command no longer removes all unused data. This behavior + can be fine-tuned by new options, like the acceptable amount of unused space or + the maximum size of data to reorganize. For more details, please see https://restic.readthedocs.io/en/stable/060_forget.html . - Moreover, `prune` now accepts the `--dry-run` option and also running `forget --dry-run - --prune` will show what `prune` would do. + Moreover, `prune` now accepts the `--dry-run` option and also running `forget + --dry-run --prune` will show what `prune` would do. This enhancement also fixes several open issues, e.g.: - https://github.com/restic/restic/issues/1140 - @@ -2598,68 +2839,74 @@ restic users. The changes are ordered by importance. * Enhancement #2941: Speed up the repacking step of the `prune` command - The repack step of the `prune` command, which moves still used file parts into new pack files - such that the old ones can be garbage collected later on, now processes multiple pack files in - parallel. This is especially beneficial for high latency backends or when using a fast network - connection. + The repack step of the `prune` command, which moves still used file parts into + new pack files such that the old ones can be garbage collected later on, now + processes multiple pack files in parallel. This is especially beneficial for + high latency backends or when using a fast network connection. https://github.com/restic/restic/pull/2941 * Enhancement #2944: Add `backup` options `--files-from-{verbatim,raw}` - The new `backup` options `--files-from-verbatim` and `--files-from-raw` read a list of - files to back up from a file. Unlike the existing `--files-from` option, these options do not - interpret the listed filenames as glob patterns; instead, whitespace in filenames is - preserved as-is and no pattern expansion is done. Please see the documentation for specifics. + The new `backup` options `--files-from-verbatim` and `--files-from-raw` read a + list of files to back up from a file. Unlike the existing `--files-from` option, + these options do not interpret the listed filenames as glob patterns; instead, + whitespace in filenames is preserved as-is and no pattern expansion is done. + Please see the documentation for specifics. - These new options are highly recommended over `--files-from`, when using a script to generate - the list of files to back up. + These new options are highly recommended over `--files-from`, when using a + script to generate the list of files to back up. https://github.com/restic/restic/issues/2944 https://github.com/restic/restic/issues/3013 * Enhancement #3006: Speed up the `rebuild-index` command - We've optimized the `rebuild-index` command. Now, existing index entries are used to - minimize the number of pack files that must be read. This speeds up the index rebuild a lot. + We've optimized the `rebuild-index` command. Now, existing index entries are + used to minimize the number of pack files that must be read. This speeds up the + index rebuild a lot. - Additionally, the option `--read-all-packs` has been added, implementing the previous - behavior. + Additionally, the option `--read-all-packs` has been added, implementing the + previous behavior. https://github.com/restic/restic/pull/3006 https://github.com/restic/restic/issue/2547 * Enhancement #3048: Add more checks for index and pack files in the `check` command - The `check` command run with the `--read-data` or `--read-data-subset` options used to only - verify only the pack file content - it did not check if the blobs within the pack are correctly - contained in the index. + The `check` command run with the `--read-data` or `--read-data-subset` options + used to only verify only the pack file content - it did not check if the blobs + within the pack are correctly contained in the index. A check for the latter is now in place, which can print the following error: Blob ID is not contained in index or position is incorrect - Another test is also added, which compares pack file sizes computed from the index and the pack - header with the actual file size. This test is able to detect truncated pack files. + Another test is also added, which compares pack file sizes computed from the + index and the pack header with the actual file size. This test is able to detect + truncated pack files. - If the index is not correct, it can be rebuilt by using the `rebuild-index` command. + If the index is not correct, it can be rebuilt by using the `rebuild-index` + command. - Having added these tests, `restic check` is now able to detect non-existing blobs which are - wrongly referenced in the index. This situation could have lead to missing data. + Having added these tests, `restic check` is now able to detect non-existing + blobs which are wrongly referenced in the index. This situation could have lead + to missing data. https://github.com/restic/restic/pull/3048 https://github.com/restic/restic/pull/3082 * Enhancement #3083: Allow usage of deprecated S3 `ListObjects` API - Some S3 API implementations, e.g. Ceph before version 14.2.5, have a broken `ListObjectsV2` - implementation which causes problems for restic when using their API endpoints. When a broken - server implementation is used, restic prints errors similar to the following: + Some S3 API implementations, e.g. Ceph before version 14.2.5, have a broken + `ListObjectsV2` implementation which causes problems for restic when using their + API endpoints. When a broken server implementation is used, restic prints errors + similar to the following: List() returned error: Truncated response should have continuation token set - As a temporary workaround, restic now allows using the older `ListObjects` endpoint by - setting the `s3.list-objects-v1` extended option, for instance: + As a temporary workaround, restic now allows using the older `ListObjects` + endpoint by setting the `s3.list-objects-v1` extended option, for instance: Restic -o s3.list-objects-v1=true snapshots @@ -2670,28 +2917,30 @@ restic users. The changes are ordered by importance. * Enhancement #3099: Reduce memory usage of `check` command - The `check` command now requires less memory if it is run without the `--check-unused` option. + The `check` command now requires less memory if it is run without the + `--check-unused` option. https://github.com/restic/restic/pull/3099 * Enhancement #3106: Parallelize scan of snapshot content in `copy` and `prune` - The `copy` and `prune` commands used to traverse the directories of snapshots one by one to find - used data. This snapshot traversal is now parallelized which can speed up this step several - times. + The `copy` and `prune` commands used to traverse the directories of snapshots + one by one to find used data. This snapshot traversal is now parallized which + can speed up this step several times. - In addition the `check` command now reports how many snapshots have already been processed. + In addition the `check` command now reports how many snapshots have already been + processed. https://github.com/restic/restic/pull/3106 * Enhancement #3130: Parallelize reading of locks and snapshots - Restic used to read snapshots sequentially. For repositories containing many snapshots this - slowed down commands which have to read all snapshots. + Restic used to read snapshots sequentially. For repositories containing many + snapshots this slowed down commands which have to read all snapshots. - Now the reading of snapshots is parallelized. This speeds up for example `prune`, `backup` and - other commands that search for snapshots with certain properties or which have to find the - `latest` snapshot. + Now the reading of snapshots is parallelized. This speeds up for example + `prune`, `backup` and other commands that search for snapshots with certain + properties or which have to find the `latest` snapshot. The speed up also applies to locks stored in the backup repository. @@ -2700,37 +2949,39 @@ restic users. The changes are ordered by importance. * Enhancement #3147: Support additional environment variables for Swift authentication - The `swift` backend now supports the following additional environment variables for passing - authentication details to restic: `OS_USER_ID`, `OS_USER_DOMAIN_ID`, + The `swift` backend now supports the following additional environment variables + for passing authentication details to restic: `OS_USER_ID`, `OS_USER_DOMAIN_ID`, `OS_PROJECT_DOMAIN_ID` and `OS_TRUST_ID` - Depending on the `openrc` configuration file these might be required when the user and project - domains differ from one another. + Depending on the `openrc` configuration file these might be required when the + user and project domains differ from one another. https://github.com/restic/restic/issues/3147 https://github.com/restic/restic/pull/3158 * Enhancement #3191: Add release binaries for MIPS architectures - We've added a few new architectures for Linux to the release binaries: `mips`, `mipsle`, - `mips64`, and `mip64le`. MIPS is mostly used for low-end embedded systems. + We've added a few new architectures for Linux to the release binaries: `mips`, + `mipsle`, `mips64`, and `mip64le`. MIPS is mostly used for low-end embedded + systems. https://github.com/restic/restic/issues/3191 https://github.com/restic/restic/pull/3208 * Enhancement #3250: Add several more error checks - We've added a lot more error checks in places where errors were previously ignored (as hinted by - the static analysis program `errcheck` via `golangci-lint`). + We've added a lot more error checks in places where errors were previously + ignored (as hinted by the static analysis program `errcheck` via + `golangci-lint`). https://github.com/restic/restic/pull/3250 * Enhancement #3254: Enable HTTP/2 for backend connections - Go's HTTP library usually automatically chooses between HTTP/1.x and HTTP/2 depending on - what the server supports. But for compatibility this mechanism is disabled if DialContext is - used (which is the case for restic). This change allows restic's HTTP client to negotiate - HTTP/2 if supported by the server. + Go's HTTP library usually automatically chooses between HTTP/1.x and HTTP/2 + depending on what the server supports. But for compatibility this mechanism is + disabled if DialContext is used (which is the case for restic). This change + allows restic's HTTP client to negotiate HTTP/2 if supported by the server. https://github.com/restic/restic/pull/3254 @@ -2761,11 +3012,11 @@ restic users. The changes are ordered by importance. * Bugfix #1212: Restore timestamps and permissions on intermediate directories - When using the `--include` option of the restore command, restic restored timestamps and - permissions only on directories selected by the include pattern. Intermediate directories, - which are necessary to restore files located in sub- directories, were created with default - permissions. We've fixed the restore command to restore timestamps and permissions for these - directories as well. + When using the `--include` option of the restore command, restic restored + timestamps and permissions only on directories selected by the include pattern. + Intermediate directories, which are necessary to restore files located in sub- + directories, were created with default permissions. We've fixed the restore + command to restore timestamps and permissions for these directories as well. https://github.com/restic/restic/issues/1212 https://github.com/restic/restic/issues/1402 @@ -2773,13 +3024,15 @@ restic users. The changes are ordered by importance. * Bugfix #1756: Mark repository files as read-only when using the local backend - Files stored in a local repository were marked as writable on the filesystem for non-Windows - systems, which did not prevent accidental file modifications outside of restic. In addition, - the local backend did not work with certain filesystems and network mounts which do not permit - modifications of file permissions. + Files stored in a local repository were marked as writeable on the filesystem + for non-Windows systems, which did not prevent accidental file modifications + outside of restic. In addition, the local backend did not work with certain + filesystems and network mounts which do not permit modifications of file + permissions. - Restic now marks files stored in a local repository as read-only on the filesystem on - non-Windows systems. The error handling is improved to support more filesystems. + Restic now marks files stored in a local repository as read-only on the + filesystem on non-Windows systems. The error handling is improved to support + more filesystems. https://github.com/restic/restic/issues/1756 https://github.com/restic/restic/issues/2157 @@ -2787,8 +3040,9 @@ restic users. The changes are ordered by importance. * Bugfix #2241: Hide password in REST backend repository URLs - When using a password in the REST backend repository URL, the password could in some cases be - included in the output from restic, e.g. when initializing a repo or during an error. + When using a password in the REST backend repository URL, the password could in + some cases be included in the output from restic, e.g. when initializing a repo + or during an error. The password is now replaced with "***" where applicable. @@ -2797,10 +3051,11 @@ restic users. The changes are ordered by importance. * Bugfix #2319: Correctly dump directories into tar files - The dump command previously wrote directories in a tar file in a way which can cause - compatibility problems. This caused, for example, 7zip on Windows to not open tar files - containing directories. In addition it was not possible to dump directories with extended - attributes. These compatibility problems are now corrected. + The dump command previously wrote directories in a tar file in a way which can + cause compatibility problems. This caused, for example, 7zip on Windows to not + open tar files containing directories. In addition it was not possible to dump + directories with extended attributes. These compatibility problems are now + corrected. In addition, a tar file now includes the name of the owner and group of a file. @@ -2809,17 +3064,18 @@ restic users. The changes are ordered by importance. * Bugfix #2491: Don't require `self-update --output` placeholder file - `restic self-update --output /path/to/new-restic` used to require that new-restic was an - existing file, to be overwritten. Now it's possible to download an updated restic binary to a - new path, without first having to create a placeholder file. + `restic self-update --output /path/to/new-restic` used to require that + new-restic was an existing file, to be overwritten. Now it's possible to + download an updated restic binary to a new path, without first having to create + a placeholder file. https://github.com/restic/restic/issues/2491 https://github.com/restic/restic/pull/2937 * Bugfix #2834: Fix rare cases of backup command hanging forever - We've fixed an issue with the backup progress reporting which could cause restic to hang - forever right before finishing a backup. + We've fixed an issue with the backup progress reporting which could cause restic + to hang forever right before finishing a backup. https://github.com/restic/restic/issues/2834 https://github.com/restic/restic/pull/2963 @@ -2833,46 +3089,50 @@ restic users. The changes are ordered by importance. * Bugfix #2942: Make --exclude-larger-than handle disappearing files - There was a small bug in the backup command's --exclude-larger-than option where files that - disappeared between scanning and actually backing them up to the repository caused a panic. - This is now fixed. + There was a small bug in the backup command's --exclude-larger-than option where + files that disappeared between scanning and actually backing them up to the + repository caused a panic. This is now fixed. https://github.com/restic/restic/issues/2942 * Bugfix #2951: Restic generate, help and self-update no longer check passwords - The commands `restic cache`, `generate`, `help` and `self-update` don't need passwords, but - they previously did run the RESTIC_PASSWORD_COMMAND (if set in the environment), prompting - users to authenticate for no reason. They now skip running the password command. + The commands `restic cache`, `generate`, `help` and `self-update` don't need + passwords, but they previously did run the RESTIC_PASSWORD_COMMAND (if set in + the environment), prompting users to authenticate for no reason. They now skip + running the password command. https://github.com/restic/restic/issues/2951 https://github.com/restic/restic/pull/2987 * Bugfix #2979: Make snapshots --json output [] instead of null when no snapshots - Restic previously output `null` instead of `[]` for the `--json snapshots` command, when - there were no snapshots in the repository. This caused some minor problems when parsing the - output, but is now fixed such that `[]` is output when the list of snapshots is empty. + Restic previously output `null` instead of `[]` for the `--json snapshots` + command, when there were no snapshots in the repository. This caused some minor + problems when parsing the output, but is now fixed such that `[]` is output when + the list of snapshots is empty. https://github.com/restic/restic/issues/2979 https://github.com/restic/restic/pull/2984 * Enhancement #340: Add support for Volume Shadow Copy Service (VSS) on Windows - Volume Shadow Copy Service allows read access to files that are locked by another process using - an exclusive lock through a filesystem snapshot. Restic was unable to backup those files - before. This update enables backing up these files. + Volume Shadow Copy Service allows read access to files that are locked by + another process using an exclusive lock through a filesystem snapshot. Restic + was unable to backup those files before. This update enables backing up these + files. - This needs to be enabled explicitly using the --use-fs-snapshot option of the backup command. + This needs to be enabled explicitely using the --use-fs-snapshot option of the + backup command. https://github.com/restic/restic/issues/340 https://github.com/restic/restic/pull/2274 * Enhancement #1458: New option --repository-file - We've added a new command-line option --repository-file as an alternative to -r. This allows - to read the repository URL from a file in order to prevent certain types of information leaks, - especially for URLs containing credentials. + We've added a new command-line option --repository-file as an alternative to -r. + This allows to read the repository URL from a file in order to prevent certain + types of information leaks, especially for URLs containing credentials. https://github.com/restic/restic/issues/1458 https://github.com/restic/restic/issues/2900 @@ -2880,27 +3140,29 @@ restic users. The changes are ordered by importance. * Enhancement #2849: Authenticate to Google Cloud Storage with access token - When using the GCS backend, it is now possible to authenticate with OAuth2 access tokens - instead of a credentials file by setting the GOOGLE_ACCESS_TOKEN environment variable. + When using the GCS backend, it is now possible to authenticate with OAuth2 + access tokens instead of a credentials file by setting the GOOGLE_ACCESS_TOKEN + environment variable. https://github.com/restic/restic/pull/2849 * Enhancement #2969: Optimize check for unchanged files during backup - During a backup restic skips processing files which have not changed since the last backup run. - Previously this required opening each file once which can be slow on network filesystems. The - backup command now checks for file changes before opening a file. This considerably reduces - the time to create a backup on network filesystems. + During a backup restic skips processing files which have not changed since the + last backup run. Previously this required opening each file once which can be + slow on network filesystems. The backup command now checks for file changes + before opening a file. This considerably reduces the time to create a backup on + network filesystems. https://github.com/restic/restic/issues/2969 https://github.com/restic/restic/pull/2970 * Enhancement #2978: Warn if parent snapshot cannot be loaded during backup - During a backup restic uses the parent snapshot to check whether a file was changed and has to be - backed up again. For this check the backup has to read the directories contained in the old - snapshot. If a tree blob cannot be loaded, restic now warns about this problem with the backup - repository. + During a backup restic uses the parent snapshot to check whether a file was + changed and has to be backed up again. For this check the backup has to read the + directories contained in the old snapshot. If a tree blob cannot be loaded, + restic now warns about this problem with the backup repository. https://github.com/restic/restic/pull/2978 @@ -2960,15 +3222,16 @@ restic users. The changes are ordered by importance. * Bugfix #1863: Report correct number of directories processed by backup - The directory statistics calculation was fixed to report the actual number of processed - directories instead of always zero. + The directory statistics calculation was fixed to report the actual number of + processed directories instead of always zero. https://github.com/restic/restic/issues/1863 * Bugfix #2254: Fix tar issues when dumping `/` - We've fixed an issue with dumping either `/` or files on the first sublevel e.g. `/foo` to tar. - This also fixes tar dumping issues on Windows where this issue could also happen. + We've fixed an issue with dumping either `/` or files on the first sublevel e.g. + `/foo` to tar. This also fixes tar dumping issues on Windows where this issue + could also happen. https://github.com/restic/restic/issues/2254 https://github.com/restic/restic/issues/2357 @@ -2976,59 +3239,63 @@ restic users. The changes are ordered by importance. * Bugfix #2281: Handle format verbs like '%' properly in `find` output - The JSON or "normal" output of the `find` command can now deal with file names that contain - substrings which the Golang `fmt` package considers "format verbs" like `%s`. + The JSON or "normal" output of the `find` command can now deal with file names + that contain substrings which the Golang `fmt` package considers "format verbs" + like `%s`. https://github.com/restic/restic/issues/2281 * Bugfix #2298: Do not hang when run as a background job - Restic did hang on exit while restoring the terminal configuration when it was started as a - background job, for example using `restic ... &`. This has been fixed by only restoring the - terminal configuration when restic is interrupted while reading a password from the - terminal. + Restic did hang on exit while restoring the terminal configuration when it was + started as a background job, for example using `restic ... &`. This has been + fixed by only restoring the terminal configuration when restic is interrupted + while reading a password from the terminal. https://github.com/restic/restic/issues/2298 * Bugfix #2389: Fix mangled json output of backup command - We've fixed a race condition in the json output of the backup command that could cause multiple - lines to get mixed up. We've also ensured that the backup summary is printed last. + We've fixed a race condition in the json output of the backup command that could + cause multiple lines to get mixed up. We've also ensured that the backup summary + is printed last. https://github.com/restic/restic/issues/2389 https://github.com/restic/restic/pull/2545 * Bugfix #2390: Refresh lock timestamp - Long-running operations did not refresh lock timestamp, resulting in locks becoming stale. - This is now fixed. + Long-running operations did not refresh lock timestamp, resulting in locks + becoming stale. This is now fixed. https://github.com/restic/restic/issues/2390 * Bugfix #2429: Backup --json reports total_bytes_processed as 0 - We've fixed the json output of total_bytes_processed. The non-json output was already fixed - with pull request #2138 but left the json output untouched. + We've fixed the json output of total_bytes_processed. The non-json output was + already fixed with pull request #2138 but left the json output untouched. https://github.com/restic/restic/issues/2429 * Bugfix #2469: Fix incorrect bytes stats in `diff` command - In some cases, the wrong number of bytes (e.g. 16777215.998 TiB) were reported by the `diff` - command. This is now fixed. + In some cases, the wrong number of bytes (e.g. 16777215.998 TiB) were reported + by the `diff` command. This is now fixed. https://github.com/restic/restic/issues/2469 * Bugfix #2518: Do not crash with Synology NAS sftp server - It was found that when restic is used to store data on an sftp server on a Synology NAS with a - relative path (one which does not start with a slash), it may go into an endless loop trying to - create directories on the server. We've fixed this bug by using a function in the sftp library - instead of our own implementation. + It was found that when restic is used to store data on an sftp server on a + Synology NAS with a relative path (one which does not start with a slash), it + may go into an endless loop trying to create directories on the server. We've + fixed this bug by using a function in the sftp library instead of our own + implementation. - The bug was discovered because the Synology sftp server behaves erratic with non-absolute - path (e.g. `home/restic-repo`). This can be resolved by just using an absolute path instead - (`/home/restic-repo`). We've also added a paragraph in the FAQ. + The bug was discovered because the Synology sftp server behaves erratic with + non-absolute path (e.g. `home/restic-repo`). This can be resolved by just using + an absolute path instead (`/home/restic-repo`). We've also added a paragraph in + the FAQ. https://github.com/restic/restic/issues/2518 https://github.com/restic/restic/issues/2363 @@ -3036,84 +3303,90 @@ restic users. The changes are ordered by importance. * Bugfix #2531: Fix incorrect size calculation in `stats --mode restore-size` - The restore-size mode of stats was counting hard-linked files as if they were independent. + The restore-size mode of stats was counting hard-linked files as if they were + independent. https://github.com/restic/restic/issues/2531 * Bugfix #2537: Fix incorrect file counts in `stats --mode restore-size` - The restore-size mode of stats was failing to count empty directories and some files with hard - links. + The restore-size mode of stats was failing to count empty directories and some + files with hard links. https://github.com/restic/restic/issues/2537 * Bugfix #2592: SFTP backend supports IPv6 addresses - The SFTP backend now supports IPv6 addresses natively, without relying on aliases in the - external SSH configuration. + The SFTP backend now supports IPv6 addresses natively, without relying on + aliases in the external SSH configuration. https://github.com/restic/restic/pull/2592 * Bugfix #2607: Honor RESTIC_CACHE_DIR environment variable on Mac and Windows - On Mac and Windows, the RESTIC_CACHE_DIR environment variable was ignored. This variable can - now be used on all platforms to set the directory where restic stores caches. + On Mac and Windows, the RESTIC_CACHE_DIR environment variable was ignored. This + variable can now be used on all platforms to set the directory where restic + stores caches. https://github.com/restic/restic/pull/2607 * Bugfix #2668: Don't abort the stats command when data blobs are missing - Running the stats command in the blobs-per-file mode on a repository with missing data blobs - previously resulted in a crash. + Runing the stats command in the blobs-per-file mode on a repository with missing + data blobs previously resulted in a crash. https://github.com/restic/restic/pull/2668 * Bugfix #2674: Add stricter prune error checks - Additional checks were added to the prune command in order to improve resiliency to backend, - hardware and/or networking issues. The checks now detect a few more cases where such outside - factors could potentially cause data loss. + Additional checks were added to the prune command in order to improve resiliency + to backend, hardware and/or networking issues. The checks now detect a few more + cases where such outside factors could potentially cause data loss. https://github.com/restic/restic/pull/2674 * Bugfix #2899: Fix possible crash in the progress bar of check --read-data - We've fixed a possible crash while displaying the progress bar for the check --read-data - command. The crash occurred when the length of the progress bar status exceeded the terminal - width, which only happened for very narrow terminal windows. + We've fixed a possible crash while displaying the progress bar for the check + --read-data command. The crash occurred when the length of the progress bar + status exceeded the terminal width, which only happened for very narrow terminal + windows. https://github.com/restic/restic/pull/2899 https://forum.restic.net/t/restic-rclone-pcloud-connection-issues/2963/15 * Change #1597: Honor the --no-lock flag in the mount command - The mount command now does not lock the repository if given the --no-lock flag. This allows to - mount repositories which are archived on a read only backend/filesystem. + The mount command now does not lock the repository if given the --no-lock flag. + This allows to mount repositories which are archived on a read only + backend/filesystem. https://github.com/restic/restic/issues/1597 https://github.com/restic/restic/pull/2821 * Change #2482: Remove vendored dependencies - We've removed the vendored dependencies (in the subdir `vendor/`). When building restic, the - Go compiler automatically fetches the dependencies. It will also cryptographically verify - that the correct code has been fetched by using the hashes in `go.sum` (see the link to the - documentation below). + We've removed the vendored dependencies (in the subdir `vendor/`). When building + restic, the Go compiler automatically fetches the dependencies. It will also + cryptographically verify that the correct code has been fetched by using the + hashes in `go.sum` (see the link to the documentation below). https://github.com/restic/restic/issues/2482 https://golang.org/cmd/go/#hdr-Module_downloading_and_verification * Change #2546: Return exit code 3 when failing to backup all source data - The backup command used to return a zero exit code as long as a snapshot could be created - successfully, even if some of the source files could not be read (in which case the snapshot - would contain the rest of the files). + The backup command used to return a zero exit code as long as a snapshot could + be created successfully, even if some of the source files could not be read (in + which case the snapshot would contain the rest of the files). - This made it hard for automation/scripts to detect failures/incomplete backups by looking at - the exit code. Restic now returns the following exit codes for the backup command: + This made it hard for automation/scripts to detect failures/incomplete backups + by looking at the exit code. Restic now returns the following exit codes for the + backup command: - - 0 when the command was successful - 1 when there was a fatal error (no snapshot created) - 3 when - some source data could not be read (incomplete snapshot created) + - 0 when the command was successful - 1 when there was a fatal error (no + snapshot created) - 3 when some source data could not be read (incomplete + snapshot created) https://github.com/restic/restic/issues/956 https://github.com/restic/restic/issues/2064 @@ -3123,12 +3396,12 @@ restic users. The changes are ordered by importance. * Change #2600: Update dependencies, require Go >= 1.13 - Restic now requires Go to be at least 1.13. This allows simplifications in the build process and - removing workarounds. + Restic now requires Go to be at least 1.13. This allows simplifications in the + build process and removing workarounds. - This is also probably the last version of restic still supporting mounting repositories via - fuse on macOS. The library we're using for fuse does not support macOS any more and osxfuse is not - open source any more. + This is also probably the last version of restic still supporting mounting + repositories via fuse on macOS. The library we're using for fuse does not + support macOS any more and osxfuse is not open source any more. https://github.com/bazil/fuse/issues/224 https://github.com/osxfuse/osxfuse/issues/590 @@ -3138,17 +3411,20 @@ restic users. The changes are ordered by importance. * Enhancement #323: Add command for copying snapshots between repositories - We've added a copy command, allowing you to copy snapshots from one repository to another. + We've added a copy command, allowing you to copy snapshots from one repository + to another. - Note that this process will have to read (download) and write (upload) the entire snapshot(s) - due to the different encryption keys used on the source and destination repository. Also, the - transferred files are not re-chunked, which may break deduplication between files already - stored in the destination repo and files copied there using this command. + Note that this process will have to read (download) and write (upload) the + entire snapshot(s) due to the different encryption keys used on the source and + destination repository. Also, the transferred files are not re-chunked, which + may break deduplication between files already stored in the destination repo and + files copied there using this command. - To fully support deduplication between repositories when the copy command is used, the init - command now supports the `--copy-chunker-params` option, which initializes the new - repository with identical parameters for splitting files into chunks as an already existing - repository. This allows copied snapshots to be equally deduplicated in both repositories. + To fully support deduplication between repositories when the copy command is + used, the init command now supports the `--copy-chunker-params` option, which + initializes the new repository with identical parameters for splitting files + into chunks as an already existing repository. This allows copied snapshots to + be equally deduplicated in both repositories. https://github.com/restic/restic/issues/323 https://github.com/restic/restic/pull/2606 @@ -3156,29 +3432,29 @@ restic users. The changes are ordered by importance. * Enhancement #551: Use optimized library for hash calculation of file chunks - We've switched the library used to calculate the hashes of file chunks, which are used for - deduplication, to the optimized Minio SHA-256 implementation. + We've switched the library used to calculate the hashes of file chunks, which + are used for deduplication, to the optimized Minio SHA-256 implementation. - Depending on the CPU it improves the hashing throughput by 10-30%. Modern x86 CPUs with the SHA - Extension should be about two to three times faster. + Depending on the CPU it improves the hashing throughput by 10-30%. Modern x86 + CPUs with the SHA Extension should be about two to three times faster. https://github.com/restic/restic/issues/551 https://github.com/restic/restic/pull/2709 * Enhancement #1570: Support specifying multiple host flags for various commands - Previously commands didn't take more than one `--host` or `-H` argument into account, which - could be limiting with e.g. the `forget` command. + Previously commands didn't take more than one `--host` or `-H` argument into + account, which could be limiting with e.g. the `forget` command. - The `dump`, `find`, `forget`, `ls`, `mount`, `restore`, `snapshots`, `stats` and `tag` - commands will now take into account multiple `--host` and `-H` flags. + The `dump`, `find`, `forget`, `ls`, `mount`, `restore`, `snapshots`, `stats` and + `tag` commands will now take into account multiple `--host` and `-H` flags. https://github.com/restic/restic/issues/1570 * Enhancement #1680: Optimize `restic mount` - We've optimized the FUSE implementation used within restic. `restic mount` is now more - responsive and uses less memory. + We've optimized the FUSE implementation used within restic. `restic mount` is + now more responsive and uses less memory. https://github.com/restic/restic/issues/1680 https://github.com/restic/restic/pull/2587 @@ -3192,10 +3468,11 @@ restic users. The changes are ordered by importance. * Enhancement #2175: Allow specifying user and host when creating keys - When adding a new key to the repository, the username and hostname for the new key can be - specified on the command line. This allows overriding the defaults, for example if you would - prefer to use the FQDN to identify the host or if you want to add keys for several different hosts - without having to run the key add command on those hosts. + When adding a new key to the repository, the username and hostname for the new + key can be specified on the command line. This allows overriding the defaults, + for example if you would prefer to use the FQDN to identify the host or if you + want to add keys for several different hosts without having to run the key add + command on those hosts. https://github.com/restic/restic/issues/2175 @@ -3209,15 +3486,16 @@ restic users. The changes are ordered by importance. Fixes "not enough cache capacity" error during restore: https://github.com/restic/restic/issues/2244 - NOTE: This new implementation does not guarantee order in which blobs are written to the target - files and, for example, the last blob of a file can be written to the file before any of the - preceeding file blobs. It is therefore possible to have gaps in the data written to the target - files if restore fails or interrupted by the user. + NOTE: This new implementation does not guarantee order in which blobs are + written to the target files and, for example, the last blob of a file can be + written to the file before any of the preceeding file blobs. It is therefore + possible to have gaps in the data written to the target files if restore fails + or interrupted by the user. - The implementation will try to preallocate space for the restored files on the filesystem to - prevent file fragmentation. This ensures good read performance for large files, like for - example VM images. If preallocating space is not supported by the filesystem, then this step is - silently skipped. + The implementation will try to preallocate space for the restored files on the + filesystem to prevent file fragmentation. This ensures good read performance for + large files, like for example VM images. If preallocating space is not supported + by the filesystem, then this step is silently skipped. https://github.com/restic/restic/pull/2195 https://github.com/restic/restic/pull/2893 @@ -3230,69 +3508,73 @@ restic users. The changes are ordered by importance. * Enhancement #2328: Improve speed of check command - We've improved the check command to traverse trees only once independent of whether they are - contained in multiple snapshots. The check command is now much faster for repositories with a - large number of snapshots. + We've improved the check command to traverse trees only once independent of + whether they are contained in multiple snapshots. The check command is now much + faster for repositories with a large number of snapshots. https://github.com/restic/restic/issues/2284 https://github.com/restic/restic/pull/2328 * Enhancement #2395: Ignore sync errors when operation not supported by local filesystem - The local backend has been modified to work with filesystems which doesn't support the `sync` - operation. This operation is normally used by restic to ensure that data files are fully - written to disk before continuing. + The local backend has been modified to work with filesystems which doesn't + support the `sync` operation. This operation is normally used by restic to + ensure that data files are fully written to disk before continuing. - For these limited filesystems, saving a file in the backend would previously fail with an - "operation not supported" error. This error is now ignored, which means that e.g. an SMB mount - on macOS can now be used as storage location for a repository. + For these limited filesystems, saving a file in the backend would previously + fail with an "operation not supported" error. This error is now ignored, which + means that e.g. an SMB mount on macOS can now be used as storage location for a + repository. https://github.com/restic/restic/issues/2395 https://forum.restic.net/t/sync-errors-on-mac-over-smb/1859 * Enhancement #2423: Support user@domain parsing as user - Added the ability for user@domain-like users to be authenticated over SFTP servers. + Added the ability for user@domain-like users to be authenticated over SFTP + servers. https://github.com/restic/restic/pull/2423 * Enhancement #2427: Add flag `--iexclude-file` to backup command - The backup command now supports the flag `--iexclude-file` which is a case-insensitive - version of `--exclude-file`. + The backup command now supports the flag `--iexclude-file` which is a + case-insensitive version of `--exclude-file`. https://github.com/restic/restic/issues/2427 https://github.com/restic/restic/pull/2898 * Enhancement #2569: Support excluding files by their size - The `backup` command now supports the `--exclude-larger-than` option to exclude files which - are larger than the specified maximum size. This can for example be useful to exclude - unimportant files with a large file size. + The `backup` command now supports the `--exclude-larger-than` option to exclude + files which are larger than the specified maximum size. This can for example be + useful to exclude unimportant files with a large file size. https://github.com/restic/restic/issues/2569 https://github.com/restic/restic/pull/2914 * Enhancement #2571: Self-heal missing file parts during backup of unchanged files - We've improved the resilience of restic to certain types of repository corruption. + We've improved the resilience of restic to certain types of repository + corruption. - For files that are unchanged since the parent snapshot, the backup command now verifies that - all parts of the files still exist in the repository. Parts that are missing, e.g. from a damaged - repository, are backed up again. This verification was already run for files that were - modified since the parent snapshot, but is now also done for unchanged files. + For files that are unchanged since the parent snapshot, the backup command now + verifies that all parts of the files still exist in the repository. Parts that + are missing, e.g. from a damaged repository, are backed up again. This + verification was already run for files that were modified since the parent + snapshot, but is now also done for unchanged files. - Note that restic will not backup file parts that are referenced in the index but where the actual - data is not present on disk, as this situation can only be detected by restic check. Please - ensure that you run `restic check` regularly. + Note that restic will not backup file parts that are referenced in the index but + where the actual data is not present on disk, as this situation can only be + detected by restic check. Please ensure that you run `restic check` regularly. https://github.com/restic/restic/issues/2571 https://github.com/restic/restic/pull/2827 * Enhancement #2576: Improve the chunking algorithm - We've updated the chunker library responsible for splitting files into smaller blocks. It - should improve the chunking throughput by 5-15% depending on the CPU. + We've updated the chunker library responsible for splitting files into smaller + blocks. It should improve the chunking throughput by 5-15% depending on the CPU. https://github.com/restic/restic/issues/2820 https://github.com/restic/restic/pull/2576 @@ -3300,65 +3582,68 @@ restic users. The changes are ordered by importance. * Enhancement #2598: Improve speed of diff command - We've improved the performance of the diff command when comparing snapshots with similar - content. It should run up to twice as fast as before. + We've improved the performance of the diff command when comparing snapshots with + similar content. It should run up to twice as fast as before. https://github.com/restic/restic/pull/2598 * Enhancement #2599: Slightly reduce memory usage of prune and stats commands - The prune and the stats command kept directory identifiers in memory twice while searching for - used blobs. + The prune and the stats command kept directory identifiers in memory twice while + searching for used blobs. https://github.com/restic/restic/pull/2599 * Enhancement #2733: S3 backend: Add support for WebIdentityTokenFile - We've added support for EKS IAM roles for service accounts feature to the S3 backend. + We've added support for EKS IAM roles for service accounts feature to the S3 + backend. https://github.com/restic/restic/issues/2703 https://github.com/restic/restic/pull/2733 * Enhancement #2773: Optimize handling of new index entries - Restic now uses less memory for backups which add a lot of data, e.g. large initial backups. In - addition, we've improved the stability in some edge cases. + Restic now uses less memory for backups which add a lot of data, e.g. large + initial backups. In addition, we've improved the stability in some edge cases. https://github.com/restic/restic/pull/2773 * Enhancement #2781: Reduce memory consumption of in-memory index - We've improved how the index is stored in memory. This change can reduce memory usage for large - repositories by up to 50% (depending on the operation). + We've improved how the index is stored in memory. This change can reduce memory + usage for large repositories by up to 50% (depending on the operation). https://github.com/restic/restic/pull/2781 https://github.com/restic/restic/pull/2812 * Enhancement #2786: Optimize `list blobs` command - We've changed the implementation of `list blobs` which should be now a bit faster and consume - almost no memory even for large repositories. + We've changed the implementation of `list blobs` which should be now a bit + faster and consume almost no memory even for large repositories. https://github.com/restic/restic/pull/2786 * Enhancement #2790: Optimized file access in restic mount - Reading large (> 100GiB) files from restic mountpoints is now faster, and the speedup is - greater for larger files. + Reading large (> 100GiB) files from restic mountpoints is now faster, and the + speedup is greater for larger files. https://github.com/restic/restic/pull/2790 * Enhancement #2840: Speed-up file deletion in forget, prune and rebuild-index - We've sped up the file deletion for the commands forget, prune and rebuild-index, especially - for remote repositories. Deletion was sequential before and is now run in parallel. + We've sped up the file deletion for the commands forget, prune and + rebuild-index, especially for remote repositories. Deletion was sequential + before and is now run in parallel. https://github.com/restic/restic/pull/2840 * Enhancement #2858: Support filtering snapshots by tag and path in the stats command - We've added filtering snapshots by `--tag tagList` and by `--path path` to the `stats` - command. This includes filtering of only 'latest' snapshots or all snapshots in a repository. + We've added filtering snapshots by `--tag tagList` and by `--path path` to the + `stats` command. This includes filtering of only 'latest' snapshots or all + snapshots in a repository. https://github.com/restic/restic/issues/2858 https://github.com/restic/restic/pull/2859 @@ -3385,81 +3670,85 @@ restic users. The changes are ordered by importance. * Bugfix #2063: Allow absolute path for filename when backing up from stdin - When backing up from stdin, handle directory path for `--stdin-filename`. This can be used to - specify the full path for the backed-up file. + When backing up from stdin, handle directory path for `--stdin-filename`. This + can be used to specify the full path for the backed-up file. https://github.com/restic/restic/issues/2063 * Bugfix #2174: Save files with invalid timestamps - When restic reads invalid timestamps (year is before 0000 or after 9999) it refused to read and - archive the file. We've changed the behavior and will now save modified timestamps with the - year set to either 0000 or 9999, the rest of the timestamp stays the same, so the file will be saved - (albeit with a bogus timestamp). + When restic reads invalid timestamps (year is before 0000 or after 9999) it + refused to read and archive the file. We've changed the behavior and will now + save modified timestamps with the year set to either 0000 or 9999, the rest of + the timestamp stays the same, so the file will be saved (albeit with a bogus + timestamp). https://github.com/restic/restic/issues/2174 https://github.com/restic/restic/issues/1173 * Bugfix #2249: Read fresh metadata for unmodified files - Restic took all metadata for files which were detected as unmodified, not taking into account - changed metadata (ownership, mode). This is now corrected. + Restic took all metadata for files which were detected as unmodified, not taking + into account changed metadata (ownership, mode). This is now corrected. https://github.com/restic/restic/issues/2249 https://github.com/restic/restic/pull/2252 * Bugfix #2301: Add upper bound for t in --read-data-subset=n/t - 256 is the effective maximum for t, but restic would allow larger values, leading to strange - behavior. + 256 is the effective maximum for t, but restic would allow larger values, + leading to strange behavior. https://github.com/restic/restic/issues/2301 https://github.com/restic/restic/pull/2304 * Bugfix #2321: Check errors when loading index files - Restic now checks and handles errors which occur when loading index files, the missing check - leads to odd errors (and a stack trace printed to users) later. This was reported in the forum. + Restic now checks and handles errors which occur when loading index files, the + missing check leads to odd errors (and a stack trace printed to users) later. + This was reported in the forum. https://github.com/restic/restic/pull/2321 https://forum.restic.net/t/check-rebuild-index-prune/1848/13 * Enhancement #2179: Use ctime when checking for file changes - Previously, restic only checked a file's mtime (along with other non-timestamp metadata) to - decide if a file has changed. This could cause restic to not notice that a file has changed (and - therefore continue to store the old version, as opposed to the modified version) if something - edits the file and then resets the timestamp. Restic now also checks the ctime of files, so any - modifications to a file should be noticed, and the modified file will be backed up. The ctime - check will be disabled if the --ignore-inode flag was given. + Previously, restic only checked a file's mtime (along with other non-timestamp + metadata) to decide if a file has changed. This could cause restic to not notice + that a file has changed (and therefore continue to store the old version, as + opposed to the modified version) if something edits the file and then resets the + timestamp. Restic now also checks the ctime of files, so any modifications to a + file should be noticed, and the modified file will be backed up. The ctime check + will be disabled if the --ignore-inode flag was given. - If this change causes problems for you, please open an issue, and we can look in to adding a - separate flag to disable just the ctime check. + If this change causes problems for you, please open an issue, and we can look in + to adding a seperate flag to disable just the ctime check. https://github.com/restic/restic/issues/2179 https://github.com/restic/restic/pull/2212 * Enhancement #2306: Allow multiple retries for interactive password input - Restic used to quit if the repository password was typed incorrectly once. Restic will now ask - the user again for the repository password if typed incorrectly. The user will now get three - tries to input the correct password before restic quits. + Restic used to quit if the repository password was typed incorrectly once. + Restic will now ask the user again for the repository password if typed + incorrectly. The user will now get three tries to input the correct password + before restic quits. https://github.com/restic/restic/issues/2306 * Enhancement #2330: Make `--group-by` accept both singular and plural - One can now use the values `host`/`hosts`, `path`/`paths` and `tag` / `tags` interchangeably - in the `--group-by` argument. + One can now use the values `host`/`hosts`, `path`/`paths` and `tag` / `tags` + interchangeably in the `--group-by` argument. https://github.com/restic/restic/issues/2330 * Enhancement #2350: Add option to configure S3 region - We've added a new option for setting the region when accessing an S3-compatible service. For - some providers, it is required to set this to a valid value. You can do that either by setting the - environment variable `AWS_DEFAULT_REGION` or using the option `s3.region`, e.g. like this: - `-o s3.region="us-east-1"`. + We've added a new option for setting the region when accessing an S3-compatible + service. For some providers, it is required to set this to a valid value. You + can do that either by setting the environment variable `AWS_DEFAULT_REGION` or + using the option `s3.region`, e.g. like this: `-o s3.region="us-east-1"`. https://github.com/restic/restic/pull/2350 @@ -3488,10 +3777,11 @@ restic users. The changes are ordered by importance. * Bugfix #2135: Return error when no bytes could be read from stdin - We assume that users reading backup data from stdin want to know when no data could be read, so now - restic returns an error when `backup --stdin` is called but no bytes could be read. Usually, - this means that an earlier command in a pipe has failed. The documentation was amended and now - recommends setting the `pipefail` option (`set -o pipefail`). + We assume that users reading backup data from stdin want to know when no data + could be read, so now restic returns an error when `backup --stdin` is called + but no bytes could be read. Usually, this means that an earlier command in a + pipe has failed. The documentation was amended and now recommends setting the + `pipefail` option (`set -o pipefail`). https://github.com/restic/restic/pull/2135 https://github.com/restic/restic/pull/2139 @@ -3502,84 +3792,88 @@ restic users. The changes are ordered by importance. * Bugfix #2203: Fix reading passwords from stdin - Passwords for the `init`, `key add`, and `key passwd` commands can now be read from - non-terminal stdin. + Passwords for the `init`, `key add`, and `key passwd` commands can now be read + from non-terminal stdin. https://github.com/restic/restic/issues/2203 * Bugfix #2224: Don't abort the find command when a tree can't be loaded - Change the find command so that missing trees don't result in a crash. Instead, the error is - logged to the debug log, and the tree ID is displayed along with the snapshot it belongs to. This - makes it possible to recover repositories that are missing trees by forgetting the snapshots - they are used in. + Change the find command so that missing trees don't result in a crash. Instead, + the error is logged to the debug log, and the tree ID is displayed along with + the snapshot it belongs to. This makes it possible to recover repositories that + are missing trees by forgetting the snapshots they are used in. https://github.com/restic/restic/issues/2224 * Enhancement #1895: Add case insensitive include & exclude options - The backup and restore commands now have --iexclude and --iinclude flags as case insensitive - variants of --exclude and --include. + The backup and restore commands now have --iexclude and --iinclude flags as case + insensitive variants of --exclude and --include. https://github.com/restic/restic/issues/1895 https://github.com/restic/restic/pull/2032 * Enhancement #1937: Support streaming JSON output for backup - We've added support for getting machine-readable status output during backup, just pass the - flag `--json` for `restic backup` and restic will output a stream of JSON objects which contain - the current progress. + We've added support for getting machine-readable status output during backup, + just pass the flag `--json` for `restic backup` and restic will output a stream + of JSON objects which contain the current progress. https://github.com/restic/restic/issues/1937 https://github.com/restic/restic/pull/1944 * Enhancement #2037: Add group-by option to snapshots command - We have added an option to group the output of the snapshots command, similar to the output of the - forget command. The option has been called "--group-by" and accepts any combination of the - values "host", "paths" and "tags", separated by commas. Default behavior (not specifying - --group-by) has not been changed. We have added support of the grouping to the JSON output. + We have added an option to group the output of the snapshots command, similar to + the output of the forget command. The option has been called "--group-by" and + accepts any combination of the values "host", "paths" and "tags", separated by + commas. Default behavior (not specifying --group-by) has not been changed. We + have added support of the grouping to the JSON output. https://github.com/restic/restic/issues/2037 https://github.com/restic/restic/pull/2087 * Enhancement #2124: Ability to dump folders to tar via stdout - We've added the ability to dump whole folders to stdout via the `dump` command. Restic now - requires at least Go 1.10 due to a limitation of the standard library for Go <= 1.9. + We've added the ability to dump whole folders to stdout via the `dump` command. + Restic now requires at least Go 1.10 due to a limitation of the standard library + for Go <= 1.9. https://github.com/restic/restic/issues/2123 https://github.com/restic/restic/pull/2124 * Enhancement #2139: Return error if no bytes could be read for `backup --stdin` - When restic is used to backup the output of a program, like `mysqldump | restic backup --stdin`, - it now returns an error if no bytes could be read at all. This catches the failure case when - `mysqldump` failed for some reason and did not output any data to stdout. + When restic is used to backup the output of a program, like `mysqldump | restic + backup --stdin`, it now returns an error if no bytes could be read at all. This + catches the failure case when `mysqldump` failed for some reason and did not + output any data to stdout. https://github.com/restic/restic/pull/2139 * Enhancement #2155: Add Openstack application credential auth for Swift - Since Openstack Queens Identity (auth V3) service supports an application credential auth - method. It allows to create a technical account with the limited roles. This commit adds an - application credential authentication method for the Swift backend. + Since Openstack Queens Identity (auth V3) service supports an application + credential auth method. It allows to create a technical account with the limited + roles. This commit adds an application credential authentication method for the + Swift backend. https://github.com/restic/restic/issues/2155 * Enhancement #2184: Add --json support to forget command - The forget command now supports the --json argument, outputting the information about what is - (or would-be) kept and removed from the repository. + The forget command now supports the --json argument, outputting the information + about what is (or would-be) kept and removed from the repository. https://github.com/restic/restic/issues/2184 https://github.com/restic/restic/pull/2185 * Enhancement #2205: Add --ignore-inode option to backup cmd - This option handles backup of virtual filesystems that do not keep fixed inodes for files, like - Fuse-based, pCloud, etc. Ignoring inode changes allows to consider the file as unchanged if - last modification date and size are unchanged. + This option handles backup of virtual filesystems that do not keep fixed inodes + for files, like Fuse-based, pCloud, etc. Ignoring inode changes allows to + consider the file as unchanged if last modification date and size are unchanged. https://github.com/restic/restic/issues/1631 https://github.com/restic/restic/pull/2205 @@ -3587,16 +3881,17 @@ restic users. The changes are ordered by importance. * Enhancement #2220: Add config option to set S3 storage class - The `s3.storage-class` option can be passed to restic (using `-o`) to specify the storage - class to be used for S3 objects created by restic. + The `s3.storage-class` option can be passed to restic (using `-o`) to specify + the storage class to be used for S3 objects created by restic. - The storage class is passed as-is to S3, so it needs to be understood by the API. On AWS, it can be - one of `STANDARD`, `STANDARD_IA`, `ONEZONE_IA`, `INTELLIGENT_TIERING` and - `REDUCED_REDUNDANCY`. If unspecified, the default storage class is used (`STANDARD` on - AWS). + The storage class is passed as-is to S3, so it needs to be understood by the + API. On AWS, it can be one of `STANDARD`, `STANDARD_IA`, `ONEZONE_IA`, + `INTELLIGENT_TIERING` and `REDUCED_REDUNDANCY`. If unspecified, the default + storage class is used (`STANDARD` on AWS). - You can mix storage classes in the same bucket, and the setting isn't stored in the restic - repository, so be sure to specify it with each command that writes to S3. + You can mix storage classes in the same bucket, and the setting isn't stored in + the restic repository, so be sure to specify it with each command that writes to + S3. https://github.com/restic/restic/issues/706 https://github.com/restic/restic/pull/2220 @@ -3624,19 +3919,19 @@ restic users. The changes are ordered by importance. * Bugfix #1989: Google Cloud Storage: Respect bandwidth limit - The GCS backend did not respect the bandwidth limit configured, a previous commit - accidentally removed support for it. + The GCS backend did not respect the bandwidth limit configured, a previous + commit accidentally removed support for it. https://github.com/restic/restic/issues/1989 https://github.com/restic/restic/pull/2100 * Bugfix #2040: Add host name filter shorthand flag for `stats` command - The default value for `--host` flag was set to 'H' (the shorthand version of the flag), this - caused the lookup for the latest snapshot to fail. + The default value for `--host` flag was set to 'H' (the shorthand version of the + flag), this caused the lookup for the latest snapshot to fail. - Add shorthand flag `-H` for `--host` (with empty default so if these flags are not specified the - latest snapshot will not filter by host name). + Add shorthand flag `-H` for `--host` (with empty default so if these flags are + not specified the latest snapshot will not filter by host name). Also add shorthand `-H` for `backup` command. @@ -3644,17 +3939,17 @@ restic users. The changes are ordered by importance. * Bugfix #2068: Correctly return error loading data - In one case during `prune` and `check`, an error loading data from the backend is not returned - properly. This is now corrected. + In one case during `prune` and `check`, an error loading data from the backend + is not returned properly. This is now corrected. https://github.com/restic/restic/issues/1999#issuecomment-433737921 https://github.com/restic/restic/pull/2068 * Bugfix #2095: Consistently use local time for snapshots times - By default snapshots created with restic backup were set to local time, but when the --time flag - was used the provided timestamp was parsed as UTC. With this change all snapshots times are set - to local time. + By default snapshots created with restic backup were set to local time, but when + the --time flag was used the provided timestamp was parsed as UTC. With this + change all snapshots times are set to local time. https://github.com/restic/restic/pull/2095 @@ -3663,65 +3958,70 @@ restic users. The changes are ordered by importance. This change significantly improves restore performance, especially when using high-latency remote repositories like B2. - The implementation now uses several concurrent threads to download and process multiple - remote files concurrently. To further reduce restore time, each remote file is downloaded - using a single repository request. + The implementation now uses several concurrent threads to download and process + multiple remote files concurrently. To further reduce restore time, each remote + file is downloaded using a single repository request. https://github.com/restic/restic/issues/1605 https://github.com/restic/restic/pull/1719 * Enhancement #2017: Mount: Enforce FUSE Unix permissions with allow-other - The fuse mount (`restic mount`) now lets the kernel check the permissions of the files within - snapshots (this is done through the `DefaultPermissions` FUSE option) when the option - `--allow-other` is specified. + The fuse mount (`restic mount`) now lets the kernel check the permissions of the + files within snapshots (this is done through the `DefaultPermissions` FUSE + option) when the option `--allow-other` is specified. - To restore the old behavior, we've added the `--no-default-permissions` option. This allows - all users that have access to the mount point to access all files within the snapshots. + To restore the old behavior, we've added the `--no-default-permissions` option. + This allows all users that have access to the mount point to access all files + within the snapshots. https://github.com/restic/restic/pull/2017 * Enhancement #2070: Make all commands display timestamps in local time - Restic used to drop the timezone information from displayed timestamps, it now converts - timestamps to local time before printing them so the times can be easily compared to. + Restic used to drop the timezone information from displayed timestamps, it now + converts timestamps to local time before printing them so the times can be + easily compared to. https://github.com/restic/restic/pull/2070 * Enhancement #2085: Allow --files-from to be specified multiple times - Before, restic took only the last file specified with `--files-from` into account, this is now - corrected. + Before, restic took only the last file specified with `--files-from` into + account, this is now corrected. https://github.com/restic/restic/issues/2085 https://github.com/restic/restic/pull/2086 * Enhancement #2089: Increase granularity of the "keep within" retention policy - The `keep-within` option of the `forget` command now accepts time ranges with an hourly - granularity. For example, running `restic forget --keep-within 3d12h` will keep all the - snapshots made within three days and twelve hours from the time of the latest snapshot. + The `keep-within` option of the `forget` command now accepts time ranges with an + hourly granularity. For example, running `restic forget --keep-within 3d12h` + will keep all the snapshots made within three days and twelve hours from the + time of the latest snapshot. https://github.com/restic/restic/issues/2089 https://github.com/restic/restic/pull/2090 * Enhancement #2094: Run command to get password - We've added the `--password-command` option which allows specifying a command that restic - runs every time the password for the repository is needed, so it can be integrated with a - password manager or keyring. The option can also be set via the environment variable - `$RESTIC_PASSWORD_COMMAND`. + We've added the `--password-command` option which allows specifying a command + that restic runs every time the password for the repository is needed, so it can + be integrated with a password manager or keyring. The option can also be set via + the environment variable `$RESTIC_PASSWORD_COMMAND`. https://github.com/restic/restic/pull/2094 * Enhancement #2097: Add key hinting - Added a new option `--key-hint` and corresponding environment variable `RESTIC_KEY_HINT`. - The key hint is a key ID to try decrypting first, before other keys in the repository. + Added a new option `--key-hint` and corresponding environment variable + `RESTIC_KEY_HINT`. The key hint is a key ID to try decrypting first, before + other keys in the repository. - This change will benefit repositories with many keys; if the correct key hint is supplied then - restic only needs to check one key. If the key hint is incorrect (the key does not exist, or the - password is incorrect) then restic will check all keys, as usual. + This change will benefit repositories with many keys; if the correct key hint is + supplied then restic only needs to check one key. If the key hint is incorrect + (the key does not exist, or the password is incorrect) then restic will check + all keys, as usual. https://github.com/restic/restic/issues/2097 @@ -3751,29 +4051,31 @@ restic users. The changes are ordered by importance. * Bugfix #1935: Remove truncated files from cache - When a file in the local cache is truncated, and restic tries to access data beyond the end of the - (cached) file, it used to return an error "EOF". This is now fixed, such truncated files are - removed and the data is fetched directly from the backend. + When a file in the local cache is truncated, and restic tries to access data + beyond the end of the (cached) file, it used to return an error "EOF". This is + now fixed, such truncated files are removed and the data is fetched directly + from the backend. https://github.com/restic/restic/issues/1935 * Bugfix #1978: Do not return an error when the scanner is slower than backup - When restic makes a backup, there's a background task called "scanner" which collects - information on how many files and directories are to be saved, in order to display progress - information to the user. When the backup finishes faster than the scanner, it is aborted - because the result is not needed any more. This logic contained a bug, where quitting the - scanner process was treated as an error, and caused restic to print an unhelpful error message - ("context canceled"). + When restic makes a backup, there's a background task called "scanner" which + collects information on how many files and directories are to be saved, in order + to display progress information to the user. When the backup finishes faster + than the scanner, it is aborted because the result is not needed any more. This + logic contained a bug, where quitting the scanner process was treated as an + error, and caused restic to print an unhelpful error message ("context + canceled"). https://github.com/restic/restic/issues/1978 https://github.com/restic/restic/pull/1991 * Enhancement #1766: Restore: suppress lchown errors when not running as root - Like "cp" and "rsync" do, restic now only reports errors for changing the ownership of files - during restore if it is run as root, on non-Windows operating systems. On Windows, the error - is reported as usual. + Like "cp" and "rsync" do, restic now only reports errors for changing the + ownership of files during restore if it is run as root, on non-Windows + operating systems. On Windows, the error is reported as usual. https://github.com/restic/restic/issues/1766 @@ -3781,113 +4083,118 @@ restic users. The changes are ordered by importance. We've updated the `find` command to support multiple patterns. - `restic find` is now able to list the snapshots containing a specific tree or blob, or even the - snapshots that contain blobs belonging to a given pack. A list of IDs can be given, as long as they - all have the same type. + `restic find` is now able to list the snapshots containing a specific tree or + blob, or even the snapshots that contain blobs belonging to a given pack. A list + of IDs can be given, as long as they all have the same type. - The command `find` can also display the pack IDs the blobs belong to, if the `--show-pack-id` - flag is provided. + The command `find` can also display the pack IDs the blobs belong to, if the + `--show-pack-id` flag is provided. https://github.com/restic/restic/issues/1777 https://github.com/restic/restic/pull/1780 * Enhancement #1876: Display reason why forget keeps snapshots - We've added a column to the list of snapshots `forget` keeps which details the reasons to keep a - particular snapshot. This makes debugging policies for forget much easier. Please remember - to always try things out with `--dry-run`! + We've added a column to the list of snapshots `forget` keeps which details the + reasons to keep a particuliar snapshot. This makes debugging policies for forget + much easier. Please remember to always try things out with `--dry-run`! https://github.com/restic/restic/pull/1876 * Enhancement #1891: Accept glob in paths loaded via --files-from - Before that, behaviour was different if paths were appended to command line or from a file, - because wild card characters were expanded by shell if appended to command line, but not - expanded if loaded from file. + Before that, behaviour was different if paths were appended to command line or + from a file, because wild card characters were expanded by shell if appended to + command line, but not expanded if loaded from file. https://github.com/restic/restic/issues/1891 * Enhancement #1909: Reject files/dirs by name first - The current scanner/archiver code had an architectural limitation: it always ran the - `lstat()` system call on all files and directories before a decision to include/exclude the - file/dir was made. This lead to a lot of unnecessary system calls for items that could have been - rejected by their name or path only. + The current scanner/archiver code had an architectural limitation: it always ran + the `lstat()` system call on all files and directories before a decision to + include/exclude the file/dir was made. This lead to a lot of unnecessary system + calls for items that could have been rejected by their name or path only. - We've changed the archiver/scanner implementation so that it now first rejects by name/path, - and only runs the system call on the remaining items. This reduces the number of `lstat()` - system calls a lot (depending on the exclude settings). + We've changed the archiver/scanner implementation so that it now first rejects + by name/path, and only runs the system call on the remaining items. This reduces + the number of `lstat()` system calls a lot (depending on the exclude settings). https://github.com/restic/restic/issues/1909 https://github.com/restic/restic/pull/1912 * Enhancement #1920: Vendor dependencies with Go 1.11 Modules - Until now, we've used `dep` for managing dependencies, we've now switch to using Go modules. - For users this does not change much, only if you want to compile restic without downloading - anything with Go 1.11, then you need to run: `go build -mod=vendor build.go` + Until now, we've used `dep` for managing dependencies, we've now switch to using + Go modules. For users this does not change much, only if you want to compile + restic without downloading anything with Go 1.11, then you need to run: `go + build -mod=vendor build.go` https://github.com/restic/restic/pull/1920 * Enhancement #1940: Add directory filter to ls command - The ls command can now be filtered by directories, so that only files in the given directories - will be shown. If the --recursive flag is specified, then ls will traverse subfolders and list - their files as well. + The ls command can now be filtered by directories, so that only files in the + given directories will be shown. If the --recursive flag is specified, then ls + will traverse subfolders and list their files as well. - It used to be possible to specify multiple snapshots, but that has been replaced by only one - snapshot and the possibility of specifying multiple directories. + It used to be possible to specify multiple snapshots, but that has been replaced + by only one snapshot and the possibility of specifying multiple directories. - Specifying directories constrains the walk, which can significantly speed up the listing. + Specifying directories constrains the walk, which can significantly speed up the + listing. https://github.com/restic/restic/issues/1940 https://github.com/restic/restic/pull/1941 * Enhancement #1949: Add new command `self-update` - We have added a new command called `self-update` which downloads the latest released version - of restic from GitHub and replaces the current binary with it. It does not rely on any external - program (so it'll work everywhere), but still verifies the GPG signature using the embedded - GPG public key. + We have added a new command called `self-update` which downloads the latest + released version of restic from GitHub and replaces the current binary with it. + It does not rely on any external program (so it'll work everywhere), but still + verifies the GPG signature using the embedded GPG public key. - By default, the `self-update` command is hidden behind the `selfupdate` built tag, which is - only set when restic is built using `build.go` (including official releases). The reason for - this is that downstream distributions will then not include the command by default, so users - are encouraged to use the platform-specific distribution mechanism. + By default, the `self-update` command is hidden behind the `selfupdate` built + tag, which is only set when restic is built using `build.go` (including official + releases). The reason for this is that downstream distributions will then not + include the command by default, so users are encouraged to use the + platform-specific distribution mechanism. https://github.com/restic/restic/pull/1949 * Enhancement #1953: Ls: Add JSON output support for restic ls cmd - We've implemented listing files in the repository with JSON as output, just pass `--json` as an - option to `restic ls`. This makes the output of the command machine readable. + We've implemented listing files in the repository with JSON as output, just pass + `--json` as an option to `restic ls`. This makes the output of the command + machine readable. https://github.com/restic/restic/pull/1953 * Enhancement #1962: Stream JSON output for ls command - The `ls` command now supports JSON output with the global `--json` flag, and this change - streams out JSON messages one object at a time rather than en entire array buffered in memory - before encoding. The advantage is it allows large listings to be handled efficiently. + The `ls` command now supports JSON output with the global `--json` flag, and + this change streams out JSON messages one object at a time rather than en entire + array buffered in memory before encoding. The advantage is it allows large + listings to be handled efficiently. - Two message types are printed: snapshots and nodes. A snapshot object will precede node - objects which belong to that snapshot. The `struct_type` field can be used to determine which - kind of message an object is. + Two message types are printed: snapshots and nodes. A snapshot object will + precede node objects which belong to that snapshot. The `struct_type` field can + be used to determine which kind of message an object is. https://github.com/restic/restic/pull/1962 * Enhancement #1967: Use `--host` everywhere - We now use the flag `--host` for all commands which need a host name, using `--hostname` (e.g. - for `restic backup`) still works, but will print a deprecation warning. Also, add the short - option `-H` where possible. + We now use the flag `--host` for all commands which need a host name, using + `--hostname` (e.g. for `restic backup`) still works, but will print a + deprecation warning. Also, add the short option `-H` where possible. https://github.com/restic/restic/issues/1967 * Enhancement #2028: Display size of cache directories - The `cache` command now by default shows the size of the individual cache directories. It can be - disabled with `--no-size`. + The `cache` command now by default shows the size of the individual cache + directories. It can be disabled with `--no-size`. https://github.com/restic/restic/issues/2028 https://github.com/restic/restic/pull/2033 @@ -3915,23 +4222,25 @@ restic users. The changes are ordered by importance. * Bugfix #1854: Allow saving files/dirs on different fs with `--one-file-system` - Restic now allows saving files/dirs on a different file system in a subdir correctly even when - `--one-file-system` is specified. + Restic now allows saving files/dirs on a different file system in a subdir + correctly even when `--one-file-system` is specified. The first thing the restic archiver code does is to build a tree of the target - files/directories. If it detects that a parent directory is already included (e.g. `restic - backup /foo /foo/bar/baz`), it'll ignore the latter argument. + files/directories. If it detects that a parent directory is already included + (e.g. `restic backup /foo /foo/bar/baz`), it'll ignore the latter argument. - Without `--one-file-system`, that's perfectly valid: If `/foo` is to be archived, it will - include `/foo/bar/baz`. But with `--one-file-system`, `/foo/bar/baz` may reside on a - different file system, so it won't be included with `/foo`. + Without `--one-file-system`, that's perfectly valid: If `/foo` is to be + archived, it will include `/foo/bar/baz`. But with `--one-file-system`, + `/foo/bar/baz` may reside on a different file system, so it won't be included + with `/foo`. https://github.com/restic/restic/issues/1854 https://github.com/restic/restic/pull/1855 * Bugfix #1861: Fix case-insensitive search with restic find - We've fixed the behavior for `restic find -i PATTERN`, which was broken in v0.9.1. + We've fixed the behavior for `restic find -i PATTERN`, which was broken in + v0.9.1. https://github.com/restic/restic/pull/1861 @@ -3944,21 +4253,22 @@ restic users. The changes are ordered by importance. * Bugfix #1880: Use `--cache-dir` argument for `check` command - `check` command now uses a temporary sub-directory of the specified directory if set using the - `--cache-dir` argument. If not set, the cache directory is created in the default temporary - directory as before. In either case a temporary cache is used to ensure the actual repository is - checked (rather than a local copy). + `check` command now uses a temporary sub-directory of the specified directory if + set using the `--cache-dir` argument. If not set, the cache directory is created + in the default temporary directory as before. In either case a temporary cache + is used to ensure the actual repository is checked (rather than a local copy). - The `--cache-dir` argument was not used by the `check` command, instead a cache directory was - created in the temporary directory. + The `--cache-dir` argument was not used by the `check` command, instead a cache + directory was created in the temporary directory. https://github.com/restic/restic/issues/1880 * Bugfix #1893: Return error when exclude file cannot be read - A bug was found: when multiple exclude files were passed to restic and one of them could not be - read, an error was printed and restic continued, ignoring even the existing exclude files. - Now, an error message is printed and restic aborts when an exclude file cannot be read. + A bug was found: when multiple exclude files were passed to restic and one of + them could not be read, an error was printed and restic continued, ignoring even + the existing exclude files. Now, an error message is printed and restic aborts + when an exclude file cannot be read. https://github.com/restic/restic/issues/1893 @@ -3969,9 +4279,9 @@ restic users. The changes are ordered by importance. * Enhancement #1477: S3 backend: accept AWS_SESSION_TOKEN - Before, it was not possible to use s3 backend with AWS temporary security credentials(with - AWS_SESSION_TOKEN). This change gives higher priority to credentials.EnvAWS credentials - provider. + Before, it was not possible to use s3 backend with AWS temporary security + credentials(with AWS_SESSION_TOKEN). This change gives higher priority to + credentials.EnvAWS credentials provider. https://github.com/restic/restic/issues/1477 https://github.com/restic/restic/pull/1479 @@ -3979,33 +4289,33 @@ restic users. The changes are ordered by importance. * Enhancement #1772: Add restore --verify to verify restored file content - Restore will print error message if restored file content does not match expected SHA256 - checksum + Restore will print error message if restored file content does not match + expected SHA256 checksum https://github.com/restic/restic/pull/1772 * Enhancement #1853: Add JSON output support to `restic key list` - This PR enables users to get the output of `restic key list` in JSON in addition to the existing - table format. + This PR enables users to get the output of `restic key list` in JSON in addition + to the existing table format. https://github.com/restic/restic/pull/1853 * Enhancement #1901: Update the Backblaze B2 library - We've updated the library we're using for accessing the Backblaze B2 service to 0.5.0 to - include support for upcoming so-called "application keys". With this feature, you can create - access credentials for B2 which are restricted to e.g. a single bucket or even a sub-directory - of a bucket. + We've updated the library we're using for accessing the Backblaze B2 service to + 0.5.0 to include support for upcoming so-called "application keys". With this + feature, you can create access credentials for B2 which are restricted to e.g. a + single bucket or even a sub-directory of a bucket. https://github.com/restic/restic/pull/1901 https://github.com/kurin/blazer * Enhancement #1906: Add support for B2 application keys - Restic can now use so-called "application keys" which can be created in the B2 dashboard and - were only introduced recently. In contrast to the "master key", such keys can be restricted to a - specific bucket and/or path. + Restic can now use so-called "application keys" which can be created in the B2 + dashboard and were only introduced recently. In contrast to the "master key", + such keys can be restricted to a specific bucket and/or path. https://github.com/restic/restic/issues/1906 https://github.com/restic/restic/pull/1914 @@ -4027,48 +4337,51 @@ restic users. The changes are ordered by importance. * Bugfix #1801: Add limiting bandwidth to the rclone backend - The rclone backend did not respect `--limit-upload` or `--limit-download`. Oftentimes it's - not necessary to use this, as the limiting in rclone itself should be used because it gives much - better results, but in case a remote instance of rclone is used (e.g. called via ssh), it is still - relevant to limit the bandwidth from restic to rclone. + The rclone backend did not respect `--limit-upload` or `--limit-download`. + Oftentimes it's not necessary to use this, as the limiting in rclone itself + should be used because it gives much better results, but in case a remote + instance of rclone is used (e.g. called via ssh), it is still relevant to limit + the bandwidth from restic to rclone. https://github.com/restic/restic/issues/1801 * Bugfix #1822: Allow uploading large files to MS Azure - Sometimes, restic creates files to be uploaded to the repository which are quite large, e.g. - when saving directories with many entries or very large files. The MS Azure API does not allow - uploading files larger that 256MiB directly, rather restic needs to upload them in blocks of - 100MiB. This is now implemented. + Sometimes, restic creates files to be uploaded to the repository which are quite + large, e.g. when saving directories with many entries or very large files. The + MS Azure API does not allow uploading files larger that 256MiB directly, rather + restic needs to upload them in blocks of 100MiB. This is now implemented. https://github.com/restic/restic/issues/1822 * Bugfix #1825: Correct `find` to not skip snapshots - Under certain circumstances, the `find` command was found to skip snapshots containing - directories with files to look for when the directories haven't been modified at all, and were - already printed as part of a different snapshot. This is now corrected. + Under certain circumstances, the `find` command was found to skip snapshots + containing directories with files to look for when the directories haven't been + modified at all, and were already printed as part of a different snapshot. This + is now corrected. - In addition, we've switched to our own matching/pattern implementation, so now things like - `restic find "/home/user/foo/**/main.go"` are possible. + In addition, we've switched to our own matching/pattern implementation, so now + things like `restic find "/home/user/foo/**/main.go"` are possible. https://github.com/restic/restic/issues/1825 https://github.com/restic/restic/issues/1823 * Bugfix #1833: Fix caching files on error - During `check` it may happen that different threads access the same file in the backend, which - is then downloaded into the cache only once. When that fails, only the thread which is - responsible for downloading the file signals the correct error. The other threads just assume - that the file has been downloaded successfully and then get an error when they try to access the - cached file. + During `check` it may happen that different threads access the same file in the + backend, which is then downloaded into the cache only once. When that fails, + only the thread which is responsible for downloading the file signals the + correct error. The other threads just assume that the file has been downloaded + successfully and then get an error when they try to access the cached file. https://github.com/restic/restic/issues/1833 * Bugfix #1834: Resolve deadlock - When the "scanning" process restic runs to find out how much data there is does not finish before - the backup itself is done, restic stops doing anything. This is resolved now. + When the "scanning" process restic runs to find out how much data there is does + not finish before the backup itself is done, restic stops doing anything. This + is resolved now. https://github.com/restic/restic/issues/1834 https://github.com/restic/restic/pull/1835 @@ -4096,7 +4409,7 @@ restic users. The changes are ordered by importance. * Enh #1665: Improve cache handling for `restic check` * Enh #1709: Improve messages `restic check` prints * Enh #1721: Add `cache` command to list cache dirs - * Enh #1735: Allow keeping a time range of snapshots + * Enh #1735: Allow keeping a time range of snaphots * Enh #1758: Allow saving OneDrive folders in Windows * Enh #1782: Use default AWS credentials chain for S3 backend @@ -4104,77 +4417,81 @@ restic users. The changes are ordered by importance. * Bugfix #1608: Respect time stamp for new backup when reading from stdin - When reading backups from stdin (via `restic backup --stdin`), restic now uses the time stamp - for the new backup passed in `--time`. + When reading backups from stdin (via `restic backup --stdin`), restic now uses + the time stamp for the new backup passed in `--time`. https://github.com/restic/restic/issues/1608 https://github.com/restic/restic/pull/1703 * Bugfix #1652: Ignore/remove invalid lock files - This corrects a bug introduced recently: When an invalid lock file in the repo is encountered - (e.g. if the file is empty), the code used to ignore that, but now returns the error. Now, invalid - files are ignored for the normal lock check, and removed when `restic unlock --remove-all` is - run. + This corrects a bug introduced recently: When an invalid lock file in the repo + is encountered (e.g. if the file is empty), the code used to ignore that, but + now returns the error. Now, invalid files are ignored for the normal lock check, + and removed when `restic unlock --remove-all` is run. https://github.com/restic/restic/issues/1652 https://github.com/restic/restic/pull/1653 * Bugfix #1684: Fix backend tests for rest-server - The REST server for restic now requires an explicit parameter (`--no-auth`) if no - authentication should be allowed. This is fixed in the tests. + The REST server for restic now requires an explicit parameter (`--no-auth`) if + no authentication should be allowed. This is fixed in the tests. https://github.com/restic/restic/pull/1684 * Bugfix #1730: Ignore sockets for restore - We've received a report and correct the behavior in which the restore code aborted restoring a - directory when a socket was encountered. Unix domain socket files cannot be restored (they are - created on the fly once a process starts listening). The error handling was corrected, and in - addition we're now ignoring sockets during restore. + We've received a report and correct the behavior in which the restore code + aborted restoring a directory when a socket was encountered. Unix domain socket + files cannot be restored (they are created on the fly once a process starts + listening). The error handling was corrected, and in addition we're now ignoring + sockets during restore. https://github.com/restic/restic/issues/1730 https://github.com/restic/restic/pull/1731 * Bugfix #1745: Correctly parse the argument to --tls-client-cert - Previously, the --tls-client-cert method attempt to read ARGV[1] (hardcoded) instead of the - argument that was passed to it. This has been corrected. + Previously, the --tls-client-cert method attempt to read ARGV[1] (hardcoded) + instead of the argument that was passed to it. This has been corrected. https://github.com/restic/restic/issues/1745 https://github.com/restic/restic/pull/1746 * Enhancement #549: Rework archiver code - The core archiver code and the complementary code for the `backup` command was rewritten - completely. This resolves very annoying issues such as 549. The first backup with this release - of restic will likely result in all files being re-read locally, so it will take a lot longer. The - next backup after that will be fast again. + The core archiver code and the complementary code for the `backup` command was + rewritten completely. This resolves very annoying issues such as 549. The first + backup with this release of restic will likely result in all files being re-read + locally, so it will take a lot longer. The next backup after that will be fast + again. - Basically, with the old code, restic took the last path component of each to-be-saved file or - directory as the top-level file/directory within the snapshot. This meant that when called as - `restic backup /home/user/foo`, the snapshot would contain the files in the directory - `/home/user/foo` as `/foo`. + Basically, with the old code, restic took the last path component of each + to-be-saved file or directory as the top-level file/directory within the + snapshot. This meant that when called as `restic backup /home/user/foo`, the + snapshot would contain the files in the directory `/home/user/foo` as `/foo`. - This is not the case any more with the new archiver code. Now, restic works very similar to what - `tar` does: When restic is called with an absolute path to save, then it'll preserve the - directory structure within the snapshot. For the example above, the snapshot would contain - the files in the directory within `/home/user/foo` in the snapshot. For relative - directories, it only preserves the relative path components. So `restic backup user/foo` - will save the files as `/user/foo` in the snapshot. + This is not the case any more with the new archiver code. Now, restic works very + similar to what `tar` does: When restic is called with an absolute path to save, + then it'll preserve the directory structure within the snapshot. For the example + above, the snapshot would contain the files in the directory within + `/home/user/foo` in the snapshot. For relative directories, it only preserves + the relative path components. So `restic backup user/foo` will save the files as + `/user/foo` in the snapshot. - While we were at it, the status display and notification system was completely rewritten. By - default, restic now shows which files are currently read (unless `--quiet` is specified) in a - multi-line status display. + While we were at it, the status display and notification system was completely + rewritten. By default, restic now shows which files are currently read (unless + `--quiet` is specified) in a multi-line status display. - The `backup` command also gained a new option: `--verbose`. It can be specified once (which - prints a bit more detail what restic is doing) or twice (which prints a line for each - file/directory restic encountered, together with some statistics). + The `backup` command also gained a new option: `--verbose`. It can be specified + once (which prints a bit more detail what restic is doing) or twice (which + prints a line for each file/directory restic encountered, together with some + statistics). - Another issue that was resolved is the new code only reads two files at most. The old code would - read way too many files in parallel, thereby slowing down the backup process on spinning discs a - lot. + Another issue that was resolved is the new code only reads two files at most. + The old code would read way too many files in parallel, thereby slowing down the + backup process on spinning discs a lot. https://github.com/restic/restic/issues/549 https://github.com/restic/restic/issues/1286 @@ -4196,11 +4513,11 @@ restic users. The changes are ordered by importance. * Enhancement #1433: Support UTF-16 encoding and process Byte Order Mark - On Windows, text editors commonly leave a Byte Order Mark at the beginning of the file to define - which encoding is used (oftentimes UTF-16). We've added code to support processing the BOMs in - text files, like the exclude files, the password file and the file passed via `--files-from`. - This does not apply to any file being saved in a backup, those are not touched and archived as they - are. + On Windows, text editors commonly leave a Byte Order Mark at the beginning of + the file to define which encoding is used (oftentimes UTF-16). We've added code + to support processing the BOMs in text files, like the exclude files, the + password file and the file passed via `--files-from`. This does not apply to any + file being saved in a backup, those are not touched and archived as they are. https://github.com/restic/restic/issues/1433 https://github.com/restic/restic/issues/1738 @@ -4208,9 +4525,9 @@ restic users. The changes are ordered by importance. * Enhancement #1477: Accept AWS_SESSION_TOKEN for the s3 backend - Before, it was not possible to use s3 backend with AWS temporary security credentials(with - AWS_SESSION_TOKEN). This change gives higher priority to credentials.EnvAWS credentials - provider. + Before, it was not possible to use s3 backend with AWS temporary security + credentials(with AWS_SESSION_TOKEN). This change gives higher priority to + credentials.EnvAWS credentials provider. https://github.com/restic/restic/issues/1477 https://github.com/restic/restic/pull/1479 @@ -4218,23 +4535,24 @@ restic users. The changes are ordered by importance. * Enhancement #1552: Use Google Application Default credentials - Google provide libraries to generate appropriate credentials with various fallback - sources. This change uses the library to generate our GCS client, which allows us to make use of - these extra methods. + Google provide libraries to generate appropriate credentials with various + fallback sources. This change uses the library to generate our GCS client, which + allows us to make use of these extra methods. - This should be backward compatible with previous restic behaviour while adding the - additional capabilities to auth from Google's internal metadata endpoints. For users - running restic in GCP this can make authentication far easier than it was before. + This should be backward compatible with previous restic behaviour while adding + the additional capabilities to auth from Google's internal metadata endpoints. + For users running restic in GCP this can make authentication far easier than it + was before. https://github.com/restic/restic/pull/1552 https://developers.google.com/identity/protocols/application-default-credentials * Enhancement #1561: Allow using rclone to access other services - We've added the ability to use rclone to store backup data on all backends that it supports. This - was done in collaboration with Nick, the author of rclone. You can now use it to first configure a - service, then restic manages the rest (starting and stopping rclone). For details, please see - the manual. + We've added the ability to use rclone to store backup data on all backends that + it supports. This was done in collaboration with Nick, the author of rclone. You + can now use it to first configure a service, then restic manages the rest + (starting and stopping rclone). For details, please see the manual. https://github.com/restic/restic/issues/1561 https://github.com/restic/restic/pull/1657 @@ -4242,9 +4560,9 @@ restic users. The changes are ordered by importance. * Enhancement #1648: Ignore AWS permission denied error when creating a repository - It's not possible to use s3 backend scoped to a subdirectory(with specific permissions). - Restic doesn't try to create repository in a subdirectory, when 'bucket exists' of parent - directory check fails due to permission issues. + It's not possible to use s3 backend scoped to a subdirectory(with specific + permissions). Restic doesn't try to create repository in a subdirectory, when + 'bucket exists' of parent directory check fails due to permission issues. https://github.com/restic/restic/pull/1648 @@ -4254,25 +4572,27 @@ restic users. The changes are ordered by importance. * Enhancement #1665: Improve cache handling for `restic check` - For safety reasons, restic does not use a local metadata cache for the `restic check` command, - so that data is loaded from the repository and restic can check it's in good condition. When the - cache is disabled, restic will fetch each tiny blob needed for checking the integrity using a - separate backend request. For non-local backends, that will take a long time, and depending on - the backend (e.g. B2) may also be much more expensive. + For safety reasons, restic does not use a local metadata cache for the `restic + check` command, so that data is loaded from the repository and restic can check + it's in good condition. When the cache is disabled, restic will fetch each tiny + blob needed for checking the integrity using a separate backend request. For + non-local backends, that will take a long time, and depending on the backend + (e.g. B2) may also be much more expensive. This PR adds a few commits which will change the behavior as follows: - * When `restic check` is called without any additional parameters, it will build a new cache in a - temporary directory, which is removed at the end of the check. This way, we'll get readahead for - metadata files (so restic will fetch the whole file when the first blob from the file is - requested), but all data is freshly fetched from the storage backend. This is the default - behavior and will work for almost all users. + * When `restic check` is called without any additional parameters, it will build + a new cache in a temporary directory, which is removed at the end of the check. + This way, we'll get readahead for metadata files (so restic will fetch the whole + file when the first blob from the file is requested), but all data is freshly + fetched from the storage backend. This is the default behavior and will work for + almost all users. - * When `restic check` is called with `--with-cache`, the default on-disc cache is used. This - behavior hasn't changed since the cache was introduced. + * When `restic check` is called with `--with-cache`, the default on-disc cache + is used. This behavior hasn't changed since the cache was introduced. - * When `--no-cache` is specified, restic falls back to the old behavior, and read all tiny blobs - in separate requests. + * When `--no-cache` is specified, restic falls back to the old behavior, and + read all tiny blobs in separate requests. https://github.com/restic/restic/issues/1665 https://github.com/restic/restic/issues/1694 @@ -4280,44 +4600,45 @@ restic users. The changes are ordered by importance. * Enhancement #1709: Improve messages `restic check` prints - Some messages `restic check` prints are not really errors, so from now on restic does not treat - them as errors any more and exits cleanly. + Some messages `restic check` prints are not really errors, so from now on restic + does not treat them as errors any more and exits cleanly. https://github.com/restic/restic/pull/1709 https://forum.restic.net/t/what-is-the-standard-procedure-to-follow-if-a-backup-or-restore-is-interrupted/571/2 * Enhancement #1721: Add `cache` command to list cache dirs - The command `cache` was added, it allows listing restic's cache directoriers together with - the last usage. It also allows removing old cache dirs without having to access a repo, via - `restic cache --cleanup` + The command `cache` was added, it allows listing restic's cache directoriers + together with the last usage. It also allows removing old cache dirs without + having to access a repo, via `restic cache --cleanup` https://github.com/restic/restic/issues/1721 https://github.com/restic/restic/pull/1749 - * Enhancement #1735: Allow keeping a time range of snapshots + * Enhancement #1735: Allow keeping a time range of snaphots - We've added the `--keep-within` option to the `forget` command. It instructs restic to keep - all snapshots within the given duration since the newest snapshot. For example, running - `restic forget --keep-within 5m7d` will keep all snapshots which have been made in the five - months and seven days since the latest snapshot. + We've added the `--keep-within` option to the `forget` command. It instructs + restic to keep all snapshots within the given duration since the newest + snapshot. For example, running `restic forget --keep-within 5m7d` will keep all + snapshots which have been made in the five months and seven days since the + latest snapshot. https://github.com/restic/restic/pull/1735 * Enhancement #1758: Allow saving OneDrive folders in Windows - Restic now contains a bugfix to two libraries, which allows saving OneDrive folders in - Windows. In order to use the newer versions of the libraries, the minimal version required to - compile restic is now Go 1.9. + Restic now contains a bugfix to two libraries, which allows saving OneDrive + folders in Windows. In order to use the newer versions of the libraries, the + minimal version required to compile restic is now Go 1.9. https://github.com/restic/restic/issues/1758 https://github.com/restic/restic/pull/1765 * Enhancement #1782: Use default AWS credentials chain for S3 backend - Adds support for file credentials to the S3 backend (e.g. ~/.aws/credentials), and reorders - the credentials chain for the S3 backend to match AWS's standard, which is static credentials, - env vars, credentials file, and finally remote. + Adds support for file credentials to the S3 backend (e.g. ~/.aws/credentials), + and reorders the credentials chain for the S3 backend to match AWS's standard, + which is static credentials, env vars, credentials file, and finally remote. https://github.com/restic/restic/pull/1782 @@ -4340,32 +4661,34 @@ restic users. The changes are ordered by importance. * Bugfix #1633: Fixed unexpected 'pack file cannot be listed' error - Due to a regression introduced in 0.8.2, the `rebuild-index` and `prune` commands failed to - read pack files with size of 587, 588, 589 or 590 bytes. + Due to a regression introduced in 0.8.2, the `rebuild-index` and `prune` + commands failed to read pack files with size of 587, 588, 589 or 590 bytes. https://github.com/restic/restic/issues/1633 https://github.com/restic/restic/pull/1635 * Bugfix #1638: Handle errors listing files in the backend - A user reported in the forum that restic completes a backup although a concurrent `prune` - operation was running. A few error messages were printed, but the backup was attempted and - completed successfully. No error code was returned. + A user reported in the forum that restic completes a backup although a + concurrent `prune` operation was running. A few error messages were printed, but + the backup was attempted and completed successfully. No error code was returned. - This should not happen: The repository is exclusively locked during `prune`, so when `restic - backup` is run in parallel, it should abort and return an error code instead. + This should not happen: The repository is exclusively locked during `prune`, so + when `restic backup` is run in parallel, it should abort and return an error + code instead. - It was found that the bug was in the code introduced only recently, which retries a List() - operation on the backend should that fail. It is now corrected. + It was found that the bug was in the code introduced only recently, which + retries a List() operation on the backend should that fail. It is now corrected. https://github.com/restic/restic/pull/1638 https://forum.restic.net/t/restic-backup-returns-0-exit-code-when-already-locked/484 * Bugfix #1641: Ignore files with invalid names in the repo - The release 0.8.2 introduced a bug: when restic encounters files in the repo which do not have a - valid name, it tries to load a file with a name of lots of zeroes instead of ignoring it. This is now - resolved, invalid file names are just ignored. + The release 0.8.2 introduced a bug: when restic encounters files in the repo + which do not have a valid name, it tries to load a file with a name of lots of + zeroes instead of ignoring it. This is now resolved, invalid file names are just + ignored. https://github.com/restic/restic/issues/1641 https://github.com/restic/restic/pull/1643 @@ -4373,8 +4696,9 @@ restic users. The changes are ordered by importance. * Enhancement #1497: Add --read-data-subset flag to check command - This change introduces ability to check integrity of a subset of repository data packs. This - can be used to spread integrity check of larger repositories over a period of time. + This change introduces ability to check integrity of a subset of repository data + packs. This can be used to spread integrity check of larger repositories over a + period of time. https://github.com/restic/restic/issues/1497 https://github.com/restic/restic/pull/1556 @@ -4387,21 +4711,22 @@ restic users. The changes are ordered by importance. * Enhancement #1623: Don't check for presence of files in the backend before writing - Before, all backend implementations were required to return an error if the file that is to be - written already exists in the backend. For most backends, that means making a request (e.g. via - HTTP) and returning an error when the file already exists. + Before, all backend implementations were required to return an error if the file + that is to be written already exists in the backend. For most backends, that + means making a request (e.g. via HTTP) and returning an error when the file + already exists. - This is not accurate, the file could have been created between the HTTP request testing for it, - and when writing starts, so we've relaxed this requirement, which saves one additional HTTP - request per newly added file. + This is not accurate, the file could have been created between the HTTP request + testing for it, and when writing starts, so we've relaxed this requeriment, + which saves one additional HTTP request per newly added file. https://github.com/restic/restic/pull/1623 * Enhancement #1634: Upgrade B2 client library, reduce HTTP requests - We've upgraded the B2 client library restic uses to access BackBlaze B2. This reduces the - number of HTTP requests needed to upload a new file from two to one, which should improve - throughput to B2. + We've upgraded the B2 client library restic uses to access BackBlaze B2. This + reduces the number of HTTP requests needed to upload a new file from two to one, + which should improve throughput to B2. https://github.com/restic/restic/pull/1634 @@ -4412,7 +4737,7 @@ restic users. The changes are ordered by importance. ## Summary - * Fix #1506: Limit bandwidth at the http.RoundTripper for HTTP based backends + * Fix #1506: Limit bandwith at the http.RoundTripper for HTTP based backends * Fix #1512: Restore directory permissions as the last step * Fix #1528: Correctly create missing subdirs in data/ * Fix #1589: Complete intermediate index upload @@ -4432,17 +4757,17 @@ restic users. The changes are ordered by importance. ## Details - * Bugfix #1506: Limit bandwidth at the http.RoundTripper for HTTP based backends + * Bugfix #1506: Limit bandwith at the http.RoundTripper for HTTP based backends https://github.com/restic/restic/issues/1506 https://github.com/restic/restic/pull/1511 * Bugfix #1512: Restore directory permissions as the last step - This change allows restoring into directories that were not writable during backup. Before, - restic created the directory, set the read-only mode and then failed to create files in the - directory. This change now restores the directory (with its permissions) as the very last - step. + This change allows restoring into directories that were not writable during + backup. Before, restic created the directory, set the read-only mode and then + failed to create files in the directory. This change now restores the directory + (with its permissions) as the very last step. https://github.com/restic/restic/issues/1512 https://github.com/restic/restic/pull/1536 @@ -4454,43 +4779,47 @@ restic users. The changes are ordered by importance. * Bugfix #1589: Complete intermediate index upload - After a user posted a comprehensive report of what he observed, we were able to find a bug and - correct it: During backup, restic uploads so-called "intermediate" index files. When the - backup finishes during a transfer of such an intermediate index, the upload is cancelled, but - the backup is finished without an error. This leads to an inconsistent state, where the - snapshot references data that is contained in the repo, but is not referenced in any index. + After a user posted a comprehensive report of what he observed, we were able to + find a bug and correct it: During backup, restic uploads so-called + "intermediate" index files. When the backup finishes during a transfer of such + an intermediate index, the upload is cancelled, but the backup is finished + without an error. This leads to an inconsistent state, where the snapshot + references data that is contained in the repo, but is not referenced in any + index. - The situation can be resolved by building a new index with `rebuild-index`, but looks very - confusing at first. Since all the data got uploaded to the repo successfully, there was no risk - of data loss, just minor inconvenience for our users. + The situation can be resolved by building a new index with `rebuild-index`, but + looks very confusing at first. Since all the data got uploaded to the repo + successfully, there was no risk of data loss, just minor inconvenience for our + users. https://github.com/restic/restic/pull/1589 https://forum.restic.net/t/error-loading-tree-check-prune-and-forget-gives-error-b2-backend/406 * Bugfix #1590: Strip spaces for lines read via --files-from - Leading and trailing spaces in lines read via `--files-from` are now stripped, so it behaves - the same as with lines read via `--exclude-file`. + Leading and trailing spaces in lines read via `--files-from` are now stripped, + so it behaves the same as with lines read via `--exclude-file`. https://github.com/restic/restic/issues/1590 https://github.com/restic/restic/pull/1613 * Bugfix #1594: Google Cloud Storage: Use generic HTTP transport - It was discovered that the Google Cloud Storage backend did not use the generic HTTP transport, - so things such as bandwidth limiting with `--limit-upload` did not work. This is resolved now. + It was discovered that the Google Cloud Storage backend did not use the generic + HTTP transport, so things such as bandwidth limiting with `--limit-upload` did + not work. This is resolved now. https://github.com/restic/restic/pull/1594 * Bugfix #1595: Backup: Remove bandwidth display - This commit removes the bandwidth displayed during backup process. It is misleading and - seldom correct, because it's neither the "read bandwidth" (only for the very first backup) nor - the "upload bandwidth". Many users are confused about (and rightly so), c.f. #1581, #1033, - #1591 + This commit removes the bandwidth displayed during backup process. It is + misleading and seldomly correct, because it's neither the "read bandwidth" (only + for the very first backup) nor the "upload bandwidth". Many users are confused + about (and rightly so), c.f. #1581, #1033, #1591 - We'll eventually replace this display with something more relevant when the new archiver code - is ready. + We'll eventually replace this display with something more relevant when the new + archiver code is ready. https://github.com/restic/restic/pull/1595 @@ -4500,59 +4829,61 @@ restic users. The changes are ordered by importance. * Enhancement #1522: Add support for TLS client certificate authentication - Support has been added for using a TLS client certificate for authentication to HTTP based - backend. A file containing the PEM encoded private key and certificate can be set using the - `--tls-client-cert` option. + Support has been added for using a TLS client certificate for authentication to + HTTP based backend. A file containing the PEM encoded private key and + certificate can be set using the `--tls-client-cert` option. https://github.com/restic/restic/issues/1522 https://github.com/restic/restic/pull/1524 * Enhancement #1538: Reduce memory allocations for querying the index - This change reduces the internal memory allocations when the index data structures in memory - are queried if a blob (part of a file) already exists in the repo. It should speed up backup a bit, - and maybe even reduce RAM usage. + This change reduces the internal memory allocations when the index data + structures in memory are queried if a blob (part of a file) already exists in + the repo. It should speed up backup a bit, and maybe even reduce RAM usage. https://github.com/restic/restic/pull/1538 * Enhancement #1541: Reduce number of remote requests during repository check - This change eliminates redundant remote repository calls and significantly improves - repository check time. + This change eliminates redundant remote repository calls and significantly + improves repository check time. https://github.com/restic/restic/issues/1541 https://github.com/restic/restic/pull/1548 * Enhancement #1549: Speed up querying across indices and scanning existing files - This change increases the whenever a blob (part of a file) is searched for in a restic - repository. This will reduce cpu usage some when backing up files already backed up by restic. - Cpu usage is further decreased when scanning files. + This change increases the whenever a blob (part of a file) is searched for in a + restic repository. This will reduce cpu usage some when backing up files already + backed up by restic. Cpu usage is further decreased when scanning files. https://github.com/restic/restic/pull/1549 * Enhancement #1554: Fuse/mount: Correctly handle EOF, add template option - We've added the `--snapshot-template` string, which can be used to specify a template for a - snapshot directory. In addition, accessing data after the end of a file via the fuse mount is now - handled correctly. + We've added the `--snapshot-template` string, which can be used to specify a + template for a snapshot directory. In addition, accessing data after the end of + a file via the fuse mount is now handled correctly. https://github.com/restic/restic/pull/1554 * Enhancement #1564: Don't terminate ssh on SIGINT - We've reworked the code which runs the `ssh` login for the sftp backend so that it can prompt for a - password (if needed) but does not exit when the user presses CTRL+C (SIGINT) e.g. during - backup. This allows restic to properly shut down when it receives SIGINT and remove the lock - file from the repo, afterwards exiting the `ssh` process. + We've reworked the code which runs the `ssh` login for the sftp backend so that + it can prompt for a password (if needed) but does not exit when the user presses + CTRL+C (SIGINT) e.g. during backup. This allows restic to properly shut down + when it receives SIGINT and remove the lock file from the repo, afterwards + exiting the `ssh` process. https://github.com/restic/restic/pull/1564 https://github.com/restic/restic/pull/1588 * Enhancement #1567: Reduce number of backend requests for rebuild-index and prune - We've found a way to reduce then number of backend requests for the `rebuild-index` and `prune` - operations. This significantly speeds up the operations for high-latency backends. + We've found a way to reduce then number of backend requests for the + `rebuild-index` and `prune` operations. This significantly speeds up the + operations for high-latency backends. https://github.com/restic/restic/issues/1567 https://github.com/restic/restic/pull/1574 @@ -4564,10 +4895,11 @@ restic users. The changes are ordered by importance. * Enhancement #1584: Limit index file size - Before, restic would create a single new index file on `prune` or `rebuild-index`, this may - lead to memory problems when this huge index is created and loaded again. We're now limiting the - size of the index file, and split newly created index files into several smaller ones. This - allows restic to be more memory-efficient. + Before, restic would create a single new index file on `prune` or + `rebuild-index`, this may lead to memory problems when this huge index is + created and loaded again. We're now limiting the size of the index file, and + split newly created index files into several smaller ones. This allows restic to + be more memory-efficient. https://github.com/restic/restic/issues/1412 https://github.com/restic/restic/issues/979 @@ -4593,8 +4925,8 @@ restic users. The changes are ordered by importance. * Bugfix #1454: Correct cache dir location for Windows and Darwin - The cache directory on Windows and Darwin was not correct, instead the directory `.cache` was - used. + The cache directory on Windows and Darwin was not correct, instead the directory + `.cache` was used. https://github.com/restic/restic/pull/1454 @@ -4605,9 +4937,9 @@ restic users. The changes are ordered by importance. * Bugfix #1459: Disable handling SIGPIPE - We've disabled handling SIGPIPE again. Turns out, writing to broken TCP connections also - raised SIGPIPE, so restic exits on the first write to a broken connection. Instead, restic - should retry the request. + We've disabled handling SIGPIPE again. Turns out, writing to broken TCP + connections also raised SIGPIPE, so restic exits on the first write to a broken + connection. Instead, restic should retry the request. https://github.com/restic/restic/issues/1457 https://github.com/restic/restic/issues/1466 @@ -4615,16 +4947,18 @@ restic users. The changes are ordered by importance. * Change #1452: Do not save atime by default - By default, the access time for files and dirs is not saved any more. It is not possible to - reliably disable updating the access time during a backup, so for the next backup the access - time is different again. This means a lot of metadata is saved. If you want to save the access time - anyway, pass `--with-atime` to the `backup` command. + By default, the access time for files and dirs is not saved any more. It is not + possible to reliably disable updating the access time during a backup, so for + the next backup the access time is different again. This means a lot of metadata + is saved. If you want to save the access time anyway, pass `--with-atime` to the + `backup` command. https://github.com/restic/restic/pull/1452 * Enhancement #11: Add the `diff` command - The command `diff` was added, it allows comparing two snapshots and listing all differences. + The command `diff` was added, it allows comparing two snapshots and listing all + differences. https://github.com/restic/restic/issues/11 https://github.com/restic/restic/issues/1460 @@ -4632,17 +4966,18 @@ restic users. The changes are ordered by importance. * Enhancement #1436: Add code to detect old cache directories - We've added code to detect old cache directories of repositories that haven't been used in a - long time, restic now prints a note when it detects that such dirs exist. Also, the option - `--cleanup-cache` was added to automatically remove such directories. That's not a problem - because the cache will be rebuild once a repo is accessed again. + We've added code to detect old cache directories of repositories that haven't + been used in a long time, restic now prints a note when it detects that such + dirs exist. Also, the option `--cleanup-cache` was added to automatically remove + such directories. That's not a problem because the cache will be rebuild once a + repo is accessed again. https://github.com/restic/restic/pull/1436 * Enhancement #1439: Improve cancellation logic - The cancellation logic was improved, restic can now shut down cleanly when requested to do so - (e.g. via ctrl+c). + The cancellation logic was improved, restic can now shut down cleanly when + requested to do so (e.g. via ctrl+c). https://github.com/restic/restic/pull/1439 @@ -4677,17 +5012,18 @@ restic users. The changes are ordered by importance. * Security #1445: Prevent writing outside the target directory during restore - A vulnerability was found in the restic restorer, which allowed attackers in special - circumstances to restore files to a location outside of the target directory. Due to the - circumstances we estimate this to be a low-risk vulnerability, but urge all users to upgrade to - the latest version of restic. + A vulnerability was found in the restic restorer, which allowed attackers in + special circumstances to restore files to a location outside of the target + directory. Due to the circumstances we estimate this to be a low-risk + vulnerability, but urge all users to upgrade to the latest version of restic. - Exploiting the vulnerability requires a Linux/Unix system which saves backups via restic and - a Windows systems which restores files from the repo. In addition, the attackers need to be able - to create files with arbitrary names which are then saved to the restic repo. For example, by - creating a file named "..\test.txt" (which is a perfectly legal filename on Linux) and - restoring a snapshot containing this file on Windows, it would be written to the parent of the - target directory. + Exploiting the vulnerability requires a Linux/Unix system which saves backups + via restic and a Windows systems which restores files from the repo. In + addition, the attackers need to be able to create files with arbitrary names + which are then saved to the restic repo. For example, by creating a file named + "..\test.txt" (which is a perfectly legal filename on Linux) and restoring a + snapshot containing this file on Windows, it would be written to the parent of + the target directory. We'd like to thank Tyler Spivey for reporting this responsibly! @@ -4695,34 +5031,36 @@ restic users. The changes are ordered by importance. * Bugfix #1256: Re-enable workaround for S3 backend - We've re-enabled a workaround for `minio-go` (the library we're using to access s3 backends), - this reduces memory usage. + We've re-enabled a workaround for `minio-go` (the library we're using to access + s3 backends), this reduces memory usage. https://github.com/restic/restic/issues/1256 https://github.com/restic/restic/pull/1267 * Bugfix #1291: Reuse backend TCP connections to BackBlaze B2 - A bug was discovered in the library we're using to access Backblaze, it now reuses already - established TCP connections which should be a lot faster and not cause network failures any - more. + A bug was discovered in the library we're using to access Backblaze, it now + reuses already established TCP connections which should be a lot faster and not + cause network failures any more. https://github.com/restic/restic/issues/1291 https://github.com/restic/restic/pull/1301 * Bugfix #1317: Run prune when `forget --prune` is called with just snapshot IDs - A bug in the `forget` command caused `prune` not to be run when `--prune` was specified without a - policy, e.g. when only snapshot IDs that should be forgotten are listed manually. + A bug in the `forget` command caused `prune` not to be run when `--prune` was + specified without a policy, e.g. when only snapshot IDs that should be forgotten + are listed manually. https://github.com/restic/restic/pull/1317 * Bugfix #1437: Remove implicit path `/restic` for the s3 backend - The s3 backend used the subdir `restic` within a bucket if no explicit path after the bucket name - was specified. Since this version, restic does not use this default path any more. If you - created a repo on s3 in a bucket without specifying a path within the bucket, you need to add - `/restic` at the end of the repository specification to access your repo: + The s3 backend used the subdir `restic` within a bucket if no explicit path + after the bucket name was specified. Since this version, restic does not use + this default path any more. If you created a repo on s3 in a bucket without + specifying a path within the bucket, you need to add `/restic` at the end of the + repository specification to access your repo: `s3:s3.amazonaws.com/bucket/restic` https://github.com/restic/restic/issues/1292 @@ -4730,32 +5068,35 @@ restic users. The changes are ordered by importance. * Enhancement #448: Sftp backend prompts for password - The sftp backend now prompts for the password if a password is necessary for login. + The sftp backend now prompts for the password if a password is necessary for + login. https://github.com/restic/restic/issues/448 https://github.com/restic/restic/pull/1270 * Enhancement #510: Add `dump` command - We've added the `dump` command which prints a file from a snapshot to stdout. This can e.g. be - used to restore files read with `backup --stdin`. + We've added the `dump` command which prints a file from a snapshot to stdout. + This can e.g. be used to restore files read with `backup --stdin`. https://github.com/restic/restic/issues/510 https://github.com/restic/restic/pull/1346 * Enhancement #1040: Add local metadata cache - We've added a local cache for metadata so that restic doesn't need to load all metadata - (snapshots, indexes, ...) from the repo each time it starts. By default the cache is active, but - there's a new global option `--no-cache` that can be used to disable the cache. By default, the - cache a standard cache folder for the OS, which can be overridden with `--cache-dir`. The cache - will automatically populate, indexes and snapshots are saved as they are loaded. Cache - directories for repos that haven't been used recently can automatically be removed by restic + We've added a local cache for metadata so that restic doesn't need to load all + metadata (snapshots, indexes, ...) from the repo each time it starts. By default + the cache is active, but there's a new global option `--no-cache` that can be + used to disable the cache. By deafult, the cache a standard cache folder for the + OS, which can be overridden with `--cache-dir`. The cache will automatically + populate, indexes and snapshots are saved as they are loaded. Cache directories + for repos that haven't been used recently can automatically be removed by restic with the `--cleanup-cache` option. - A related change was to by default create pack files in the repo that contain either data or - metadata, not both mixed together. This allows easy caching of only the metadata files. The - next run of `restic prune` will untangle mixed files automatically. + A related change was to by default create pack files in the repo that contain + either data or metadata, not both mixed together. This allows easy caching of + only the metadata files. The next run of `restic prune` will untangle mixed + files automatically. https://github.com/restic/restic/issues/29 https://github.com/restic/restic/issues/738 @@ -4767,8 +5108,8 @@ restic users. The changes are ordered by importance. * Enhancement #1102: Add subdirectory `ids` to fuse mount - The fuse mount now has an `ids` subdirectory which contains the snapshots below their (short) - IDs. + The fuse mount now has an `ids` subdirectory which contains the snapshots below + their (short) IDs. https://github.com/restic/restic/issues/1102 https://github.com/restic/restic/pull/1299 @@ -4776,17 +5117,17 @@ restic users. The changes are ordered by importance. * Enhancement #1114: Add `--cacert` to specify TLS certificates to check against - We've added the `--cacert` option which can be used to pass one (or more) CA certificates to - restic. These are used in addition to the system CA certificates to verify HTTPS certificates - (e.g. for the REST backend). + We've added the `--cacert` option which can be used to pass one (or more) CA + certificates to restic. These are used in addition to the system CA certificates + to verify HTTPS certificates (e.g. for the REST backend). https://github.com/restic/restic/issues/1114 https://github.com/restic/restic/pull/1276 * Enhancement #1216: Add upload/download limiting - We've added support for rate limiting through `--limit-upload` and `--limit-download` - flags. + We've added support for rate limiting through `--limit-upload` and + `--limit-download` flags. https://github.com/restic/restic/issues/1216 https://github.com/restic/restic/pull/1336 @@ -4794,15 +5135,15 @@ restic users. The changes are ordered by importance. * Enhancement #1249: Add `latest` symlink in fuse mount - The directory structure in the fuse mount now exposes a symlink `latest` which points to the - latest snapshot in that particular directory. + The directory structure in the fuse mount now exposes a symlink `latest` which + points to the latest snapshot in that particular directory. https://github.com/restic/restic/pull/1249 * Enhancement #1269: Add `--compact` to `forget` command - The option `--compact` was added to the `forget` command to provide the same compact view as the - `snapshots` command. + The option `--compact` was added to the `forget` command to provide the same + compact view as the `snapshots` command. https://github.com/restic/restic/pull/1269 @@ -4815,25 +5156,26 @@ restic users. The changes are ordered by importance. * Enhancement #1274: Add `generate` command, replaces `manpage` and `autocomplete` - The `generate` command has been added, which replaces the now removed commands `manpage` and - `autocomplete`. This release of restic contains the most recent manpages in `doc/man` and the - auto-completion files for bash and zsh in `doc/bash-completion.sh` and - `doc/zsh-completion.zsh` + The `generate` command has been added, which replaces the now removed commands + `manpage` and `autocomplete`. This release of restic contains the most recent + manpages in `doc/man` and the auto-completion files for bash and zsh in + `doc/bash-completion.sh` and `doc/zsh-completion.zsh` https://github.com/restic/restic/issues/1274 https://github.com/restic/restic/pull/1282 * Enhancement #1281: Google Cloud Storage backend needs less permissions - The Google Cloud Storage backend no longer requires the service account to have the - `storage.buckets.get` permission ("Storage Admin" role) in `restic init` if the bucket - already exists. + The Google Cloud Storage backend no longer requires the service account to have + the `storage.buckets.get` permission ("Storage Admin" role) in `restic init` if + the bucket already exists. https://github.com/restic/restic/pull/1281 * Enhancement #1319: Make `check` print `no errors found` explicitly - The `check` command now explicitly prints `No errors were found` when no errors could be found. + The `check` command now explicetly prints `No errors were found` when no errors + could be found. https://github.com/restic/restic/issues/1303 https://github.com/restic/restic/pull/1319 @@ -4844,8 +5186,8 @@ restic users. The changes are ordered by importance. * Enhancement #1367: Allow comments in files read from via `--file-from` - When the list of files/dirs to be saved is read from a file with `--files-from`, comment lines - (starting with `#`) are now ignored. + When the list of files/dirs to be saved is read from a file with `--files-from`, + comment lines (starting with `#`) are now ignored. https://github.com/restic/restic/issues/1367 https://github.com/restic/restic/pull/1368 @@ -4863,9 +5205,10 @@ restic users. The changes are ordered by importance. * Bugfix #1246: List all files stored in Google Cloud Storage - For large backups stored in Google Cloud Storage, the `prune` command fails because listing - only returns the first 1000 files. This has been corrected, no data is lost in the process. In - addition, a plausibility check was added to `prune`. + For large backups stored in Google Cloud Storage, the `prune` command fails + because listing only returns the first 1000 files. This has been corrected, no + data is lost in the process. In addition, a plausibility check was added to + `prune`. https://github.com/restic/restic/issues/1246 https://github.com/restic/restic/pull/1247 @@ -4903,26 +5246,28 @@ restic users. The changes are ordered by importance. * Bugfix #1167: Do not create a local repo unless `init` is used - When a restic command other than `init` is used with a local repository and the repository - directory does not exist, restic creates the directory structure. That's an error, only the - `init` command should create the dir. + When a restic command other than `init` is used with a local repository and the + repository directory does not exist, restic creates the directory structure. + That's an error, only the `init` command should create the dir. https://github.com/restic/restic/issues/1167 https://github.com/restic/restic/pull/1182 * Bugfix #1191: Make sure to write profiling files on interrupt - Since a few releases restic had the ability to write profiling files for memory and CPU usage - when `debug` is enabled. It was discovered that when restic is interrupted (ctrl+c is - pressed), the proper shutdown hook is not run. This is now corrected. + Since a few releases restic had the ability to write profiling files for memory + and CPU usage when `debug` is enabled. It was discovered that when restic is + interrupted (ctrl+c is pressed), the proper shutdown hook is not run. This is + now corrected. https://github.com/restic/restic/pull/1191 * Enhancement #317: Add `--exclude-caches` and `--exclude-if-present` - A new option `--exclude-caches` was added that allows excluding cache directories (that are - tagged as such). This is a special case of a more generic option `--exclude-if-present` which - excludes a directory if a file with a specific name (and contents) is present. + A new option `--exclude-caches` was added that allows excluding cache + directories (that are tagged as such). This is a special case of a more generic + option `--exclude-if-present` which excludes a directory if a file with a + specific name (and contents) is present. https://github.com/restic/restic/issues/317 https://github.com/restic/restic/pull/1170 @@ -4943,16 +5288,17 @@ restic users. The changes are ordered by importance. * Enhancement #1126: Use the standard Go git repository layout, use `dep` for vendoring - The git repository layout was changed to resemble the layout typically used in Go projects, - we're not using `gb` for building restic any more and vendoring the dependencies is now taken - care of by `dep`. + The git repository layout was changed to resemble the layout typically used in + Go projects, we're not using `gb` for building restic any more and vendoring the + dependencies is now taken care of by `dep`. https://github.com/restic/restic/pull/1126 * Enhancement #1132: Make `key` command always prompt for a password - The `key` command now prompts for a password even if the original password to access a repo has - been specified via the `RESTIC_PASSWORD` environment variable or a password file. + The `key` command now prompts for a password even if the original password to + access a repo has been specified via the `RESTIC_PASSWORD` environment variable + or a password file. https://github.com/restic/restic/issues/1132 https://github.com/restic/restic/pull/1133 @@ -4969,8 +5315,8 @@ restic users. The changes are ordered by importance. * Enhancement #1149: Add support for storing backups on Microsoft Azure Blob Storage - The library we're using to access the service requires Go 1.8, so restic now needs at least Go - 1.8. + The library we're using to access the service requires Go 1.8, so restic now + needs at least Go 1.8. https://github.com/restic/restic/issues/609 https://github.com/restic/restic/pull/1149 @@ -4996,8 +5342,8 @@ restic users. The changes are ordered by importance. * Enhancement #1218: Add `--compact` to `snapshots` command - The option `--compact` was added to the `snapshots` command to get a better overview of the - snapshots in a repo. It limits each snapshot to a single line. + The option `--compact` was added to the `snapshots` command to get a better + overview of the snapshots in a repo. It limits each snapshot to a single line. https://github.com/restic/restic/issues/1218 https://github.com/restic/restic/pull/1223 @@ -5021,18 +5367,19 @@ restic users. The changes are ordered by importance. * Bugfix #1115: Fix `prune`, only include existing files in indexes - A bug was found (and corrected) in the index rebuilding after prune, which led to indexes which - include blobs that were not present in the repo any more. There were already checks in place - which detected this situation and aborted with an error message. A new run of either `prune` or - `rebuild-index` corrected the index files. This is now fixed and a test has been added to detect - this. + A bug was found (and corrected) in the index rebuilding after prune, which led + to indexes which include blobs that were not present in the repo any more. There + were already checks in place which detected this situation and aborted with an + error message. A new run of either `prune` or `rebuild-index` corrected the + index files. This is now fixed and a test has been added to detect this. https://github.com/restic/restic/pull/1115 * Enhancement #1055: Create subdirs below `data/` for local/sftp backends - The local and sftp backends now create the subdirs below `data/` on open/init. This way, restic - makes sure that they always exist. This is connected to an issue for the sftp server. + The local and sftp backends now create the subdirs below `data/` on open/init. + This way, restic makes sure that they always exist. This is connected to an + issue for the sftp server. https://github.com/restic/restic/issues/1055 https://github.com/restic/rest-server/pull/11#issuecomment-309879710 @@ -5041,17 +5388,18 @@ restic users. The changes are ordered by importance. * Enhancement #1067: Allow loading credentials for s3 from IAM - When no S3 credentials are specified in the environment variables, restic now tries to load - credentials from an IAM instance profile when the s3 backend is used. + When no S3 credentials are specified in the environment variables, restic now + tries to load credentials from an IAM instance profile when the s3 backend is + used. https://github.com/restic/restic/issues/1067 https://github.com/restic/restic/pull/1086 * Enhancement #1073: Add `migrate` cmd to migrate from `s3legacy` to `default` layout - The `migrate` command for changing the `s3legacy` layout to the `default` layout for s3 - backends has been improved: It can now be restarted with `restic migrate --force s3_layout` - and automatically retries operations on error. + The `migrate` command for changing the `s3legacy` layout to the `default` layout + for s3 backends has been improved: It can now be restarted with `restic migrate + --force s3_layout` and automatically retries operations on error. https://github.com/restic/restic/issues/1073 https://github.com/restic/restic/pull/1075 @@ -5091,18 +5439,18 @@ restic users. The changes are ordered by importance. * Bugfix #965: Switch to `default` repo layout for the s3 backend - The default layout for the s3 backend is now `default` (instead of `s3legacy`). Also, there's a - new `migrate` command to convert an existing repo, it can be run like this: `restic migrate - s3_layout` + The default layout for the s3 backend is now `default` (instead of `s3legacy`). + Also, there's a new `migrate` command to convert an existing repo, it can be run + like this: `restic migrate s3_layout` https://github.com/restic/restic/issues/965 https://github.com/restic/restic/pull/1004 * Bugfix #1013: Switch back to using the high-level minio-go API for s3 - For the s3 backend we're back to using the high-level API the s3 client library for uploading - data, a few users reported dropped connections (which the library will automatically retry - now). + For the s3 backend we're back to using the high-level API the s3 client library + for uploading data, a few users reported dropped connections (which the library + will automatically retry now). https://github.com/restic/restic/issues/1013 https://github.com/restic/restic/issues/1023 @@ -5115,9 +5463,10 @@ restic users. The changes are ordered by importance. * Enhancement #636: Add dirs `tags` and `hosts` to fuse mount - The fuse mount now has two more directories: `tags` contains a subdir for each tag, which in turn - contains only the snapshots that have this tag. The subdir `hosts` contains a subdir for each - host that has a snapshot, and the subdir contains the snapshots for that host. + The fuse mount now has two more directories: `tags` contains a subdir for each + tag, which in turn contains only the snapshots that have this tag. The subdir + `hosts` contains a subdir for each host that has a snapshot, and the subdir + contains the snapshots for that host. https://github.com/restic/restic/issues/636 https://github.com/restic/restic/pull/1050 @@ -5129,8 +5478,9 @@ restic users. The changes are ordered by importance. * Enhancement #989: Improve performance of the `find` command - Improved performance for the `find` command: Restic recognizes paths it has already checked - for the files in question, so the number of backend requests is reduced a lot. + Improved performance for the `find` command: Restic recognizes paths it has + already checked for the files in question, so the number of backend requests is + reduced a lot. https://github.com/restic/restic/issues/989 https://github.com/restic/restic/pull/993 @@ -5143,16 +5493,17 @@ restic users. The changes are ordered by importance. * Enhancement #1021: Detect invalid backend name and print error - Restic now tries to detect when an invalid/unknown backend is used and returns an error - message. + Restic now tries to detect when an invalid/unknown backend is used and returns + an error message. https://github.com/restic/restic/issues/1021 https://github.com/restic/restic/pull/1070 * Enhancement #1029: Remove invalid pack files when `prune` is run - The `prune` command has been improved and will now remove invalid pack files, for example files - that have not been uploaded completely because a backup was interrupted. + The `prune` command has been improved and will now remove invalid pack files, + for example files that have not been uploaded completely because a backup was + interrupted. https://github.com/restic/restic/issues/1029 https://github.com/restic/restic/pull/1036 @@ -5172,24 +5523,24 @@ restic users. The changes are ordered by importance. * Enhancement #974: Remove regular status reports - Regular status report: We've removed the status report that was printed every 10 seconds when - restic is run non-interactively. You can still force reporting the current status by sending a - `USR1` signal to the process. + Regular status report: We've removed the status report that was printed every 10 + seconds when restic is run non-interactively. You can still force reporting the + current status by sending a `USR1` signal to the process. https://github.com/restic/restic/pull/974 * Enhancement #981: Remove temporary path from binary in `build.go` - The `build.go` now strips the temporary directory used for compilation from the binary. This - is the first step in enabling reproducible builds. + The `build.go` now strips the temporary directory used for compilation from the + binary. This is the first step in enabling reproducible builds. https://github.com/restic/restic/pull/981 * Enhancement #985: Allow multiple parallel idle HTTP connections - Backends based on HTTP now allow several idle connections in parallel. This is especially - important for the REST backend, which (when used with a local server) may create a lot - connections and exhaust available ports quickly. + Backends based on HTTP now allow several idle connections in parallel. This is + especially important for the REST backend, which (when used with a local server) + may create a lot connections and exhaust available ports quickly. https://github.com/restic/restic/issues/985 https://github.com/restic/restic/pull/986 @@ -5209,21 +5560,22 @@ restic users. The changes are ordered by importance. * Enhancement #957: Make `forget` consistent - The `forget` command was corrected to be more consistent in which snapshots are to be - forgotten. It is possible that the new code removes more snapshots than before, so please - review what would be deleted by using the `--dry-run` option. + The `forget` command was corrected to be more consistent in which snapshots are + to be forgotten. It is possible that the new code removes more snapshots than + before, so please review what would be deleted by using the `--dry-run` option. https://github.com/restic/restic/issues/953 https://github.com/restic/restic/pull/957 * Enhancement #962: Improve memory and runtime for the s3 backend - We've updated the library used for accessing s3, switched to using a lower level API and added - caching for some requests. This lead to a decrease in memory usage and a great speedup. In - addition, we added benchmark functions for all backends, so we can track improvements over - time. The Continuous Integration test service we're using (Travis) now runs the s3 backend - tests not only against a Minio server, but also against the Amazon s3 live service, so we should - be notified of any regressions much sooner. + We've updated the library used for accessing s3, switched to using a lower level + API and added caching for some requests. This lead to a decrease in memory usage + and a great speedup. In addition, we added benchmark functions for all backends, + so we can track improvements over time. The Continuous Integration test service + we're using (Travis) now runs the s3 backend tests not only against a Minio + server, but also against the Amazon s3 live service, so we should be notified of + any regressions much sooner. https://github.com/restic/restic/pull/962 https://github.com/restic/restic/pull/960 @@ -5233,11 +5585,12 @@ restic users. The changes are ordered by importance. * Enhancement #966: Unify repository layout for all backends - Up to now the s3 backend used a special repository layout. We've decided to unify the repository - layout and implemented the default layout also for the s3 backend. For creating a new - repository on s3 with the default layout, use `restic -o s3.layout=default init`. For further - commands the option is not necessary any more, restic will automatically detect the correct - layout to use. A future version will switch to the default layout for new repositories. + Up to now the s3 backend used a special repository layout. We've decided to + unify the repository layout and implemented the default layout also for the s3 + backend. For creating a new repository on s3 with the default layout, use + `restic -o s3.layout=default init`. For further commands the option is not + necessary any more, restic will automatically detect the correct layout to use. + A future version will switch to the default layout for new repositories. https://github.com/restic/restic/issues/965 https://github.com/restic/restic/pull/966 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4318a2107..dc278fa3a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,8 @@ Ways to Help Out Thank you for your contribution! Please **open an issue first** (or add a comment to an existing issue) if you plan to work on any code or add a new feature. This way, duplicate work is prevented and we can discuss your ideas -and design first. +and design first. Small bugfixes are an exception to this rule, just open a +pull request in this case. There are several ways you can help us out. First of all code contributions and bug fixes are most welcome. However even "minor" details as fixing spelling diff --git a/VERSION b/VERSION index 201a22c8f..5f2491c5a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.16.2 +0.16.4 diff --git a/changelog/unreleased/issue-4560 b/changelog/0.16.3_2024-01-14/issue-4560 similarity index 88% rename from changelog/unreleased/issue-4560 rename to changelog/0.16.3_2024-01-14/issue-4560 index c421f6e69..4346019d5 100644 --- a/changelog/unreleased/issue-4560 +++ b/changelog/0.16.3_2024-01-14/issue-4560 @@ -4,7 +4,7 @@ Since Go 1.21, most filesystem reparse points on Windows are considered to be irregular files. This caused restic to show an `error: invalid node type ""` error message for those files. -We have improved the error message to include the file path for those files: +This error message has now been improved and includes the relevant file path: `error: nodeFromFileInfo path/to/file: unsupported file type "irregular"`. As irregular files are not required to behave like regular files, it is not possible to provide a generic way to back up those files. diff --git a/changelog/0.16.3_2024-01-14/issue-4574 b/changelog/0.16.3_2024-01-14/issue-4574 new file mode 100644 index 000000000..bfb41620e --- /dev/null +++ b/changelog/0.16.3_2024-01-14/issue-4574 @@ -0,0 +1,11 @@ +Bugfix: Support backup of deduplicated files on Windows again + +With the official release builds of restic 0.16.1 and 0.16.2, it was not +possible to back up files that were deduplicated by the corresponding +Windows Server feature. This also applied to restic versions built using +Go 1.21.0-1.21.4. + +The Go version used to build restic has now been updated to fix this. + +https://github.com/restic/restic/issues/4574 +https://github.com/restic/restic/pull/4621 diff --git a/changelog/unreleased/issue-4612 b/changelog/0.16.3_2024-01-14/issue-4612 similarity index 100% rename from changelog/unreleased/issue-4612 rename to changelog/0.16.3_2024-01-14/issue-4612 diff --git a/changelog/unreleased/pull-4624 b/changelog/0.16.3_2024-01-14/pull-4624 similarity index 69% rename from changelog/unreleased/pull-4624 rename to changelog/0.16.3_2024-01-14/pull-4624 index fbdbb1558..6fff3c9f9 100644 --- a/changelog/unreleased/pull-4624 +++ b/changelog/0.16.3_2024-01-14/pull-4624 @@ -1,11 +1,11 @@ -Bugfix: Correct restore progress information if an error occurs +Bugfix: Correct `restore` progress information if an error occurs -If an error occurred while restoring a snapshot, this could cause the restore +If an error occurred while restoring a snapshot, this could cause the `restore` progress bar to show incorrect information. In addition, if a data file could not be loaded completely, then errors would also be reported for some already restored files. -We have improved the error reporting of the restore command to be more accurate. +Error reporting of the `restore` command has now been made more accurate. https://github.com/restic/restic/pull/4624 https://forum.restic.net/t/errors-restoring-with-restic-on-windows-server-s3/6943 diff --git a/changelog/unreleased/pull-4626 b/changelog/0.16.3_2024-01-14/pull-4626 similarity index 100% rename from changelog/unreleased/pull-4626 rename to changelog/0.16.3_2024-01-14/pull-4626 diff --git a/changelog/0.16.4_2024-02-04/issue-4529 b/changelog/0.16.4_2024-02-04/issue-4529 new file mode 100644 index 000000000..fed726d2d --- /dev/null +++ b/changelog/0.16.4_2024-02-04/issue-4529 @@ -0,0 +1,18 @@ +Enhancement: Add extra verification of data integrity before upload + +Hardware issues, or a bug in restic or its dependencies, could previously cause +corruption in the files restic created and stored in the repository. Detecting +such corruption previously required explicitly running the `check --read-data` +or `check --read-data-subset` commands. + +To further ensure data integrity, even in the case of hardware issues or +software bugs, restic now performs additional verification of the files about +to be uploaded to the repository. + +These extra checks will increase CPU usage during backups. They can therefore, +if absolutely necessary, be disabled using the `--no-extra-verify` global +option. Please note that this should be combined with more active checking +using the previously mentioned check commands. + +https://github.com/restic/restic/issues/4529 +https://github.com/restic/restic/pull/4681 diff --git a/changelog/0.16.4_2024-02-04/issue-4677 b/changelog/0.16.4_2024-02-04/issue-4677 new file mode 100644 index 000000000..8fa6cf65b --- /dev/null +++ b/changelog/0.16.4_2024-02-04/issue-4677 @@ -0,0 +1,19 @@ +Bugfix: Downgrade zstd library to fix rare data corruption at max. compression + +In restic 0.16.3, backups where the compression level was set to `max` (using +`--compression max`) could in rare and very specific circumstances result in +data corruption due to a bug in the library used for compressing data. Restic +0.16.1 and 0.16.2 were not affected. + +Restic now uses the previous version of the library used to compress data, the +same version used by restic 0.16.2. Please note that the `auto` compression +level (which restic uses by default) was never affected, and even if you used +`max` compression, chances of being affected by this issue are small. + +To check a repository for any corruption, run `restic check --read-data`. This +will download and verify the whole repository and can be used at any time to +completely verify the integrity of a repository. If the `check` command detects +anomalies, follow the suggested steps. + +https://github.com/restic/restic/issues/4677 +https://github.com/restic/restic/pull/4679 diff --git a/changelog/unreleased/issue-4251 b/changelog/unreleased/issue-4251 index 31be52401..d1d3f4508 100644 --- a/changelog/unreleased/issue-4251 +++ b/changelog/unreleased/issue-4251 @@ -1,14 +1,16 @@ -Enhancement: Support reading backup from a program's standard output +Enhancement: Support reading backup from a commands's standard output -When reading data from stdin, the `backup` command could not verify whether the -corresponding command completed successfully. +The `backup` command now supports the `--stdin-from-command` option. When using +this option, the arguments to `backup` are interpreted as a command instead of +paths to back up. `backup` then executes the given command and stores the +standard output from it in the backup, similar to the what the `--stdin` option +does. This also enables restic to verify that the command completes with exit +code zero. A non-zero exit code causes the backup to fail. -The `backup` command now supports starting an arbitrary command and sourcing -the backup content from its standard output. This enables restic to verify that -the command completes with exit code zero. A non-zero exit code causes the -backup to fail. +Note that the `--stdin` option does not have to be specified at the same time, +and that the `--stdin-filename` option also applies to `--stdin-from-command`. -Example: `restic backup --stdin-from-command mysqldump [...]` +Example: `restic backup --stdin-from-command --stdin-filename dump.sql mysqldump [...]` https://github.com/restic/restic/issues/4251 https://github.com/restic/restic/pull/4410 diff --git a/changelog/unreleased/issue-4549 b/changelog/unreleased/issue-4549 new file mode 100644 index 000000000..4829a9881 --- /dev/null +++ b/changelog/unreleased/issue-4549 @@ -0,0 +1,11 @@ +Enhancement: Add `--ncdu` option to `ls` command + +NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories. +It has an option to save a directory tree and analyse it later. +The `ls` command now supports the `--ncdu` option which outputs information +about a snapshot in the NCDU format. + +You can use it as follows: `restic ls latest --ncdu | ncdu -f -` + +https://github.com/restic/restic/issues/4549 +https://github.com/restic/restic/pull/4550 diff --git a/changelog/unreleased/issue-4574 b/changelog/unreleased/issue-4574 deleted file mode 100644 index 3668ae6c3..000000000 --- a/changelog/unreleased/issue-4574 +++ /dev/null @@ -1,11 +0,0 @@ -Bugfix: support backup of deduplicated files on Windows again - -With the official release builds of restic 0.16.1 and 0.16.2, it was not -possible to back up files that were deduplicated by the corresponding Windows -Server feature. This also applies to restic versions built using Go -1.21.0 - 1.21.4. - -We have updated the used Go version to fix this. - -https://github.com/restic/restic/issues/4574 -https://github.com/restic/restic/pull/4621 diff --git a/changelog/unreleased/issue-4583 b/changelog/unreleased/issue-4583 new file mode 100644 index 000000000..97b0e6ba7 --- /dev/null +++ b/changelog/unreleased/issue-4583 @@ -0,0 +1,12 @@ +Enhancement: Ignore s3.storage-class for metadata if archive tier is specified + +There is no official cold storage support in restic, use this option at your +own risk. + +Restic always stored all files on s3 using the specified `s3.storage-class`. +Now, restic will store metadata using a non-archive storage tier to avoid +problems when accessing a repository. To restore any data, it is still +necessary to manually warm up the required data beforehand. + +https://github.com/restic/restic/issues/4583 +https://github.com/restic/restic/pull/4584 diff --git a/changelog/unreleased/issue-4656 b/changelog/unreleased/issue-4656 new file mode 100644 index 000000000..8d16f0b48 --- /dev/null +++ b/changelog/unreleased/issue-4656 @@ -0,0 +1,7 @@ +Bugfix: Properly report the ID of newly added keys + +`restic key add` now reports the ID of a newly added key. This simplifies +selecting a specific key using the `--key-hint key` option. + +https://github.com/restic/restic/issues/4656 +https://github.com/restic/restic/pull/4657 diff --git a/changelog/unreleased/issue-4676 b/changelog/unreleased/issue-4676 new file mode 100644 index 000000000..e95118e72 --- /dev/null +++ b/changelog/unreleased/issue-4676 @@ -0,0 +1,8 @@ +Enhancement: Move key add, list, remove and passwd as separate sub-commands + +Restic now provides usage documentation for the `key` command. Each sub-command; +`add`, `list`, `remove` and `passwd` now have their own sub-command documentation +which can be invoked using `restic key --help`. + +https://github.com/restic/restic/issues/4676 +https://github.com/restic/restic/pull/4685 diff --git a/changelog/unreleased/issue-4678 b/changelog/unreleased/issue-4678 new file mode 100644 index 000000000..9f9a213e1 --- /dev/null +++ b/changelog/unreleased/issue-4678 @@ -0,0 +1,8 @@ +Enhancement: Add --target flag to the dump command + +Restic `dump` always printed to the standard output. It now permits to select a +`--target` file to write the output to. + +https://github.com/restic/restic/issues/4678 +https://github.com/restic/restic/pull/4682 +https://github.com/restic/restic/pull/4692 diff --git a/changelog/unreleased/pull-4611 b/changelog/unreleased/pull-4611 new file mode 100644 index 000000000..940de9c26 --- /dev/null +++ b/changelog/unreleased/pull-4611 @@ -0,0 +1,7 @@ +Enhancement: Back up windows created time and file attributes like hidden flag + +Restic did not back up windows-specific meta-data like created time and file attributes like hidden flag. +Restic now backs up file created time and file attributes like hidden, readonly and encrypted flag when backing up files and folders on windows. + +https://github.com/restic/restic/pull/4611 + diff --git a/changelog/unreleased/pull-4615 b/changelog/unreleased/pull-4615 new file mode 100644 index 000000000..7e2d4a017 --- /dev/null +++ b/changelog/unreleased/pull-4615 @@ -0,0 +1,6 @@ +Bugfix: `find` ignored directories in some cases + +In some cases, the `find` command ignored empty or moved directories. This has +been fixed. + +https://github.com/restic/restic/pull/4615 diff --git a/changelog/unreleased/pull-4644 b/changelog/unreleased/pull-4644 new file mode 100644 index 000000000..8000bce7e --- /dev/null +++ b/changelog/unreleased/pull-4644 @@ -0,0 +1,10 @@ +Enhancement: Improve `repair packs` command + +The `repair packs` command has been improved to also be able to process +truncated pack files. The `check --read-data` command will provide instructions +on using the command if necessary to repair a repository. See the guide at +https://restic.readthedocs.io/en/stable/077_troubleshooting.html for further +instructions. + +https://github.com/restic/restic/pull/4644 +https://github.com/restic/restic/pull/4655 diff --git a/changelog/unreleased/pull-4664 b/changelog/unreleased/pull-4664 new file mode 100644 index 000000000..74196cd9b --- /dev/null +++ b/changelog/unreleased/pull-4664 @@ -0,0 +1,8 @@ +Enhancement: `ls` uses `message_type` field to distinguish JSON messages + +The `ls` command was the only command that used the `struct_type` field to determine +the message type in the JSON output format. Now, the JSON output of the +`ls` command also includes the `message_type`. The `struct_type` field is +still included, but it deprecated. + +https://github.com/restic/restic/pull/4664 diff --git a/changelog/unreleased/pull-4703 b/changelog/unreleased/pull-4703 new file mode 100644 index 000000000..4df3385a0 --- /dev/null +++ b/changelog/unreleased/pull-4703 @@ -0,0 +1,9 @@ +Bugfix: Shutdown cleanly when SIGTERM is received + +Prior, if restic received SIGTERM it'd just immediately terminate skipping +cleanup- resulting in potential issues like stale locks being left behind. + +This primarily effected containerized restic invocations- they use SIGTERM- +but this could be triggered via a simple `killall restic` in addition. + +https://github.com/restic/restic/pull/4703 diff --git a/cmd/restic/cleanup.go b/cmd/restic/cleanup.go index 75933fe96..5a6cf79e1 100644 --- a/cmd/restic/cleanup.go +++ b/cmd/restic/cleanup.go @@ -19,7 +19,7 @@ var cleanupHandlers struct { func init() { cleanupHandlers.ch = make(chan os.Signal, 1) go CleanupHandler(cleanupHandlers.ch) - signal.Notify(cleanupHandlers.ch, syscall.SIGINT) + signal.Notify(cleanupHandlers.ch, syscall.SIGINT, syscall.SIGTERM) } // AddCleanupHandler adds the function f to the list of cleanup handlers so @@ -56,7 +56,7 @@ func RunCleanupHandlers(code int) int { return code } -// CleanupHandler handles the SIGINT signals. +// CleanupHandler handles the SIGINT and SIGTERM signals. func CleanupHandler(c <-chan os.Signal) { for s := range c { debug.Log("signal %v received, cleaning up", s) @@ -70,7 +70,7 @@ func CleanupHandler(c <-chan os.Signal) { code := 0 - if s == syscall.SIGINT { + if s == syscall.SIGINT || s == syscall.SIGTERM { code = 130 } else { code = 1 diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index a2b81a759..318d17796 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -12,7 +12,6 @@ import ( "runtime" "strconv" "strings" - "sync" "time" "github.com/spf13/cobra" @@ -25,7 +24,6 @@ import ( "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/textfile" - "github.com/restic/restic/internal/ui" "github.com/restic/restic/internal/ui/backup" "github.com/restic/restic/internal/ui/termstatus" ) @@ -44,7 +42,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was a fatal error (no snapshot created). Exit status is 3 if some source data could not be read (incomplete snapshot created). `, - PreRun: func(cmd *cobra.Command, args []string) { + PreRun: func(_ *cobra.Command, _ []string) { if backupOptions.Host == "" { hostname, err := os.Hostname() if err != nil { @@ -56,31 +54,9 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea }, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - var wg sync.WaitGroup - cancelCtx, cancel := context.WithCancel(ctx) - defer func() { - // shutdown termstatus - cancel() - wg.Wait() - }() - - term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet) - wg.Add(1) - go func() { - defer wg.Done() - term.Run(cancelCtx) - }() - - // use the terminal for stdout/stderr - prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr - defer func() { - globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr - }() - stdioWrapper := ui.NewStdioWrapper(term) - globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr() - - return runBackup(ctx, backupOptions, globalOptions, term, args) + term, cancel := setupTermstatus() + defer cancel() + return runBackup(cmd.Context(), backupOptions, globalOptions, term, args) }, } @@ -135,7 +111,7 @@ func init() { f.StringVar(&backupOptions.ExcludeLargerThan, "exclude-larger-than", "", "max `size` of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)") f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin") f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin") - f.BoolVar(&backupOptions.StdinCommand, "stdin-from-command", false, "execute command and store its stdout") + f.BoolVar(&backupOptions.StdinCommand, "stdin-from-command", false, "interpret arguments as command to execute and store its stdout") f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)") f.UintVar(&backupOptions.ReadConcurrency, "read-concurrency", 0, "read `n` files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)") f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag") @@ -150,7 +126,7 @@ func init() { f.StringArrayVar(&backupOptions.FilesFromRaw, "files-from-raw", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)") f.StringVar(&backupOptions.TimeStamp, "time", "", "`time` of the backup (ex. '2012-11-01 22:08:41') (default: now)") f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories") - f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files") + f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number and ctime changes when checking for modified files") f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files") f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done") f.BoolVar(&backupOptions.NoScan, "no-scan", false, "do not run scanner to estimate size of backup") @@ -435,7 +411,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er // parent returns the ID of the parent snapshot. If there is none, nil is // returned. -func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) { +func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) { if opts.Force { return nil, nil } @@ -633,7 +609,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter wg.Go(func() error { return sc.Scan(cancelCtx, targets) }) } - arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: backupOptions.ReadConcurrency}) + arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: opts.ReadConcurrency}) arch.SelectByName = selectByNameFilter arch.Select = selectFilter arch.WithAtime = opts.WithAtime diff --git a/cmd/restic/cmd_cache.go b/cmd/restic/cmd_cache.go index 4a10d1027..354cec288 100644 --- a/cmd/restic/cmd_cache.go +++ b/cmd/restic/cmd_cache.go @@ -28,7 +28,7 @@ EXIT STATUS Exit status is 0 if the command was successful, and non-zero if there was any error. `, DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, args []string) error { return runCache(cacheOptions, globalOptions, args) }, } diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index f04a4fe71..990702b61 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -38,7 +38,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er RunE: func(cmd *cobra.Command, args []string) error { return runCheck(cmd.Context(), checkOptions, globalOptions, args) }, - PreRunE: func(cmd *cobra.Command, args []string) error { + PreRunE: func(_ *cobra.Command, _ []string) error { return checkFlags(checkOptions) }, } @@ -336,20 +336,18 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args errorsFound = true Warnf("%v\n", err) if err, ok := err.(*checker.ErrPackData); ok { - if strings.Contains(err.Error(), "wrong data returned, hash is") { - salvagePacks = append(salvagePacks, err.PackID) - } + salvagePacks = append(salvagePacks, err.PackID) } } p.Done() if len(salvagePacks) > 0 { - Warnf("\nThe repository contains pack files with damaged blobs. These blobs must be removed to repair the repository. This can be done using the following commands:\n\n") - var strIds []string + Warnf("\nThe repository contains pack files with damaged blobs. These blobs must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n") + var strIDs []string for _, id := range salvagePacks { - strIds = append(strIds, id.String()) + strIDs = append(strIDs, id.String()) } - Warnf("RESTIC_FEATURES=repair-packs-v1 restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIds, " ")) + Warnf("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIDs, " ")) Warnf("Corrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n") } } diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 60413de21..a87e7a0c5 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -52,19 +52,23 @@ Exit status is 0 if the command was successful, and non-zero if there was any er }, } -var tryRepair bool -var repairByte bool -var extractPack bool -var reuploadBlobs bool +type DebugExamineOptions struct { + TryRepair bool + RepairByte bool + ExtractPack bool + ReuploadBlobs bool +} + +var debugExamineOpts DebugExamineOptions func init() { cmdRoot.AddCommand(cmdDebug) cmdDebug.AddCommand(cmdDebugDump) cmdDebug.AddCommand(cmdDebugExamine) - cmdDebugExamine.Flags().BoolVar(&extractPack, "extract-pack", false, "write blobs to the current directory") - cmdDebugExamine.Flags().BoolVar(&reuploadBlobs, "reupload-blobs", false, "reupload blobs to the repository") - cmdDebugExamine.Flags().BoolVar(&tryRepair, "try-repair", false, "try to repair broken blobs with single bit flips") - cmdDebugExamine.Flags().BoolVar(&repairByte, "repair-byte", false, "try to repair broken blobs by trying bytes") + cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.ExtractPack, "extract-pack", false, "write blobs to the current directory") + cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.ReuploadBlobs, "reupload-blobs", false, "reupload blobs to the repository") + cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.TryRepair, "try-repair", false, "try to repair broken blobs with single bit flips") + cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.RepairByte, "repair-byte", false, "try to repair broken blobs by trying bytes") } func prettyPrintJSON(wr io.Writer, item interface{}) error { @@ -133,7 +137,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) }) } -func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error { +func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer) error { return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { Printf("index_id: %v\n", id) if err != nil { @@ -196,7 +200,7 @@ var cmdDebugExamine = &cobra.Command{ Short: "Examine a pack file", DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runDebugExamine(cmd.Context(), globalOptions, args) + return runDebugExamine(cmd.Context(), globalOptions, debugExamineOpts, args) }, } @@ -315,7 +319,7 @@ func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte { return out } -func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, list []restic.Blob) error { +func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, packID restic.ID, list []restic.Blob) error { dec, err := zstd.NewReader(nil) if err != nil { panic(err) @@ -328,7 +332,7 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li wg, ctx := errgroup.WithContext(ctx) - if reuploadBlobs { + if opts.ReuploadBlobs { repo.StartPackUploader(ctx, wg) } @@ -356,8 +360,8 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li filePrefix := "" if err != nil { Warnf("error decrypting blob: %v\n", err) - if tryRepair || repairByte { - plaintext = tryRepairWithBitflip(ctx, key, buf, repairByte) + if opts.TryRepair || opts.RepairByte { + plaintext = tryRepairWithBitflip(ctx, key, buf, opts.RepairByte) } if plaintext != nil { outputPrefix = "repaired " @@ -391,13 +395,13 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li Printf(" successfully %vdecrypted blob (length %v), hash is %v, ID matches\n", outputPrefix, len(plaintext), id) prefix = "correct-" } - if extractPack { + if opts.ExtractPack { err = storePlainBlob(id, filePrefix+prefix, plaintext) if err != nil { return err } } - if reuploadBlobs { + if opts.ReuploadBlobs { _, _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true) if err != nil { return err @@ -406,7 +410,7 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li } } - if reuploadBlobs { + if opts.ReuploadBlobs { return repo.Flush(ctx) } return nil @@ -437,7 +441,7 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte) error { return nil } -func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) error { +func runDebugExamine(ctx context.Context, gopts GlobalOptions, opts DebugExamineOptions, args []string) error { repo, err := OpenRepository(ctx, gopts) if err != nil { return err @@ -476,7 +480,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er } for _, id := range ids { - err := examinePack(ctx, repo, id) + err := examinePack(ctx, opts, repo, id) if err != nil { Warnf("error: %v\n", err) } @@ -487,7 +491,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er return nil } -func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) error { +func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID) error { Printf("examine %v\n", id) h := backend.Handle{ @@ -524,7 +528,7 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro checkPackSize(blobs, fi.Size) - err = loadBlobs(ctx, repo, id, blobs) + err = loadBlobs(ctx, opts, repo, id, blobs) if err != nil { Warnf("error: %v\n", err) } else { @@ -542,7 +546,7 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro checkPackSize(blobs, fi.Size) if !blobsLoaded { - return loadBlobs(ctx, repo, id, blobs) + return loadBlobs(ctx, opts, repo, id, blobs) } return nil } diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index ea40d2860..3bd29fa67 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -61,7 +61,7 @@ func init() { f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata") } -func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) { +func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.LoaderUnpacked, desc string) (*restic.Snapshot, string, error) { sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc) if err != nil { return nil, "", errors.Fatal(err.Error()) @@ -71,7 +71,7 @@ func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, // Comparer collects all things needed to compare two snapshots. type Comparer struct { - repo restic.Repository + repo restic.BlobLoader opts DiffOptions printChange func(change *Change) } @@ -147,7 +147,7 @@ type DiffStatsContainer struct { } // updateBlobs updates the blob counters in the stats struct. -func updateBlobs(repo restic.Repository, blobs restic.BlobSet, stats *DiffStat) { +func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat) { for h := range blobs { switch h.Type { case restic.DataBlob: @@ -401,7 +401,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args [] c := &Comparer{ repo: repo, - opts: diffOptions, + opts: opts, printChange: func(change *Change) { Printf("%-5s%v\n", change.Modifier, change.Path) }, @@ -418,7 +418,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args [] } if gopts.Quiet { - c.printChange = func(change *Change) {} + c.printChange = func(_ *Change) {} } stats := &DiffStatsContainer{ diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index e6020d847..9178f2abe 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -46,6 +46,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er type DumpOptions struct { restic.SnapshotFilter Archive string + Target string } var dumpOptions DumpOptions @@ -56,6 +57,7 @@ func init() { flags := cmdDump.Flags() initSingleSnapshotFilter(flags, &dumpOptions.SnapshotFilter) flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"") + flags.StringVarP(&dumpOptions.Target, "target", "t", "", "write the output to target `path`") } func splitPath(p string) []string { @@ -67,11 +69,11 @@ func splitPath(p string) []string { return append(s, f) } -func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, d *dump.Dumper) error { +func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, canWriteArchiveFunc func() error) error { // If we print / we need to assume that there are multiple nodes at that // level in the tree. if pathComponents[0] == "" { - if err := checkStdoutArchive(); err != nil { + if err := canWriteArchiveFunc(); err != nil { return err } return d.DumpTree(ctx, tree, "/") @@ -91,9 +93,9 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor if err != nil { return errors.Wrapf(err, "cannot load subtree for %q", item) } - return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d) + return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d, canWriteArchiveFunc) case dump.IsDir(node): - if err := checkStdoutArchive(); err != nil { + if err := canWriteArchiveFunc(); err != nil { return err } subtree, err := restic.LoadTree(ctx, repo, *node.Subtree) @@ -168,8 +170,24 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] return errors.Fatalf("loading tree for snapshot %q failed: %v", snapshotIDString, err) } - d := dump.New(opts.Archive, repo, os.Stdout) - err = printFromTree(ctx, tree, repo, "/", splittedPath, d) + outputFileWriter := os.Stdout + canWriteArchiveFunc := checkStdoutArchive + + if opts.Target != "" { + file, err := os.Create(opts.Target) + if err != nil { + return fmt.Errorf("cannot dump to file: %w", err) + } + defer func() { + _ = file.Close() + }() + + outputFileWriter = file + canWriteArchiveFunc = func() error { return nil } + } + + d := dump.New(opts.Archive, repo, outputFileWriter) + err = printFromTree(ctx, tree, repo, "/", splittedPath, d, canWriteArchiveFunc) if err != nil { return errors.Fatalf("cannot dump file: %v", err) } diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index c30650823..7ea7c425a 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -126,6 +126,7 @@ func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) { // Make the following attributes disappear Name byte `json:"name,omitempty"` ExtendedAttributes byte `json:"extended_attributes,omitempty"` + GenericAttributes byte `json:"generic_attributes,omitempty"` Device byte `json:"device,omitempty"` Content byte `json:"content,omitempty"` Subtree byte `json:"subtree,omitempty"` @@ -244,13 +245,12 @@ func (s *statefulOutput) Finish() { // Finder bundles information needed to find a file or directory. type Finder struct { - repo restic.Repository - pat findPattern - out statefulOutput - ignoreTrees restic.IDSet - blobIDs map[string]struct{} - treeIDs map[string]struct{} - itemsFound int + repo restic.Repository + pat findPattern + out statefulOutput + blobIDs map[string]struct{} + treeIDs map[string]struct{} + itemsFound int } func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error { @@ -261,17 +261,17 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error } f.out.newsn = sn - return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { + return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { debug.Log("Error loading tree %v: %v", parentTreeID, err) Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID()) - return false, walker.ErrSkipNode + return walker.ErrSkipNode } if node == nil { - return false, nil + return nil } normalizedNodepath := nodepath @@ -284,7 +284,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error for _, pat := range f.pat.pattern { found, err := filter.Match(pat, normalizedNodepath) if err != nil { - return false, err + return err } if found { foundMatch = true @@ -292,16 +292,13 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error } } - var ( - ignoreIfNoMatch = true - errIfNoMatch error - ) + var errIfNoMatch error if node.Type == "dir" { var childMayMatch bool for _, pat := range f.pat.pattern { mayMatch, err := filter.ChildMatch(pat, normalizedNodepath) if err != nil { - return false, err + return err } if mayMatch { childMayMatch = true @@ -310,31 +307,28 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error } if !childMayMatch { - ignoreIfNoMatch = true errIfNoMatch = walker.ErrSkipNode - } else { - ignoreIfNoMatch = false } } if !foundMatch { - return ignoreIfNoMatch, errIfNoMatch + return errIfNoMatch } if !f.pat.oldest.IsZero() && node.ModTime.Before(f.pat.oldest) { debug.Log(" ModTime is older than %s\n", f.pat.oldest) - return ignoreIfNoMatch, errIfNoMatch + return errIfNoMatch } if !f.pat.newest.IsZero() && node.ModTime.After(f.pat.newest) { debug.Log(" ModTime is newer than %s\n", f.pat.newest) - return ignoreIfNoMatch, errIfNoMatch + return errIfNoMatch } debug.Log(" found match\n") f.out.PrintPattern(nodepath, node) - return false, nil - }) + return nil + }}) } func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { @@ -345,17 +339,17 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { } f.out.newsn = sn - return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { + return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { debug.Log("Error loading tree %v: %v", parentTreeID, err) Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID()) - return false, walker.ErrSkipNode + return walker.ErrSkipNode } if node == nil { - return false, nil + return nil } if node.Type == "dir" && f.treeIDs != nil { @@ -373,7 +367,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { // looking for blobs) if f.itemsFound >= len(f.treeIDs) && f.blobIDs == nil { // Return an error to terminate the Walk - return true, errors.New("OK") + return errors.New("OK") } } } @@ -394,8 +388,8 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { } } - return false, nil - }) + return nil + }}) } var errAllPacksFound = errors.New("all packs found") @@ -593,10 +587,9 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args [] } f := &Finder{ - repo: repo, - pat: pat, - out: statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON}, - ignoreTrees: restic.NewIDSet(), + repo: repo, + pat: pat, + out: statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON}, } if opts.BlobID { diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index a7f39dc4e..65ff449a3 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -33,7 +33,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runForget(cmd.Context(), forgetOptions, globalOptions, args) + return runForget(cmd.Context(), forgetOptions, forgetPruneOptions, globalOptions, args) }, } @@ -98,6 +98,7 @@ type ForgetOptions struct { } var forgetOptions ForgetOptions +var forgetPruneOptions PruneOptions func init() { cmdRoot.AddCommand(cmdForget) @@ -132,7 +133,7 @@ func init() { f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed") f.SortFlags = false - addPruneOptions(cmdForget) + addPruneOptions(cmdForget, &forgetPruneOptions) } func verifyForgetOptions(opts *ForgetOptions) error { @@ -151,7 +152,7 @@ func verifyForgetOptions(opts *ForgetOptions) error { return nil } -func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, args []string) error { +func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOptions, gopts GlobalOptions, args []string) error { err := verifyForgetOptions(&opts) if err != nil { return err diff --git a/cmd/restic/cmd_forget_integration_test.go b/cmd/restic/cmd_forget_integration_test.go index 8908d5a5f..1c027a240 100644 --- a/cmd/restic/cmd_forget_integration_test.go +++ b/cmd/restic/cmd_forget_integration_test.go @@ -9,5 +9,8 @@ import ( func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) { opts := ForgetOptions{} - rtest.OK(t, runForget(context.TODO(), opts, gopts, args)) + pruneOpts := PruneOptions{ + MaxUnused: "5%", + } + rtest.OK(t, runForget(context.TODO(), opts, pruneOpts, gopts, args)) } diff --git a/cmd/restic/cmd_generate.go b/cmd/restic/cmd_generate.go index b284767ca..ba710e708 100644 --- a/cmd/restic/cmd_generate.go +++ b/cmd/restic/cmd_generate.go @@ -21,7 +21,9 @@ EXIT STATUS Exit status is 0 if the command was successful, and non-zero if there was any error. `, DisableAutoGenTag: true, - RunE: runGenerate, + RunE: func(_ *cobra.Command, args []string) error { + return runGenerate(genOpts, args) + }, } type generateOptions struct { @@ -90,48 +92,48 @@ func writePowerShellCompletion(file string) error { return cmdRoot.GenPowerShellCompletionFile(file) } -func runGenerate(_ *cobra.Command, args []string) error { +func runGenerate(opts generateOptions, args []string) error { if len(args) > 0 { return errors.Fatal("the generate command expects no arguments, only options - please see `restic help generate` for usage and flags") } - if genOpts.ManDir != "" { - err := writeManpages(genOpts.ManDir) + if opts.ManDir != "" { + err := writeManpages(opts.ManDir) if err != nil { return err } } - if genOpts.BashCompletionFile != "" { - err := writeBashCompletion(genOpts.BashCompletionFile) + if opts.BashCompletionFile != "" { + err := writeBashCompletion(opts.BashCompletionFile) if err != nil { return err } } - if genOpts.FishCompletionFile != "" { - err := writeFishCompletion(genOpts.FishCompletionFile) + if opts.FishCompletionFile != "" { + err := writeFishCompletion(opts.FishCompletionFile) if err != nil { return err } } - if genOpts.ZSHCompletionFile != "" { - err := writeZSHCompletion(genOpts.ZSHCompletionFile) + if opts.ZSHCompletionFile != "" { + err := writeZSHCompletion(opts.ZSHCompletionFile) if err != nil { return err } } - if genOpts.PowerShellCompletionFile != "" { - err := writePowerShellCompletion(genOpts.PowerShellCompletionFile) + if opts.PowerShellCompletionFile != "" { + err := writePowerShellCompletion(opts.PowerShellCompletionFile) if err != nil { return err } } var empty generateOptions - if genOpts == empty { + if opts == empty { return errors.Fatal("nothing to do, please specify at least one output file/dir") } diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index e147f537e..c687eca53 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -1,266 +1,18 @@ package main import ( - "context" - "encoding/json" - "os" - "strings" - "sync" - - "github.com/restic/restic/internal/backend" - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/ui/table" - "github.com/spf13/cobra" ) var cmdKey = &cobra.Command{ - Use: "key [flags] [list|add|remove|passwd] [ID]", + Use: "key", Short: "Manage keys (passwords)", Long: ` -The "key" command manages keys (passwords) for accessing the repository. - -EXIT STATUS -=========== - -Exit status is 0 if the command was successful, and non-zero if there was any error. -`, - DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - return runKey(cmd.Context(), globalOptions, args) - }, +The "key" command allows you to set multiple access keys or passwords +per repository. + `, } -var ( - newPasswordFile string - keyUsername string - keyHostname string -) - func init() { cmdRoot.AddCommand(cmdKey) - - flags := cmdKey.Flags() - flags.StringVarP(&newPasswordFile, "new-password-file", "", "", "`file` from which to read the new password") - flags.StringVarP(&keyUsername, "user", "", "", "the username for new keys") - flags.StringVarP(&keyHostname, "host", "", "", "the hostname for new keys") -} - -func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions) error { - type keyInfo struct { - Current bool `json:"current"` - ID string `json:"id"` - UserName string `json:"userName"` - HostName string `json:"hostName"` - Created string `json:"created"` - } - - var m sync.Mutex - var keys []keyInfo - - err := restic.ParallelList(ctx, s, restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, size int64) error { - k, err := repository.LoadKey(ctx, s, id) - if err != nil { - Warnf("LoadKey() failed: %v\n", err) - return nil - } - - key := keyInfo{ - Current: id == s.KeyID(), - ID: id.Str(), - UserName: k.Username, - HostName: k.Hostname, - Created: k.Created.Local().Format(TimeFormat), - } - - m.Lock() - defer m.Unlock() - keys = append(keys, key) - return nil - }) - - if err != nil { - return err - } - - if gopts.JSON { - return json.NewEncoder(globalOptions.stdout).Encode(keys) - } - - tab := table.New() - tab.AddColumn(" ID", "{{if .Current}}*{{else}} {{end}}{{ .ID }}") - tab.AddColumn("User", "{{ .UserName }}") - tab.AddColumn("Host", "{{ .HostName }}") - tab.AddColumn("Created", "{{ .Created }}") - - for _, key := range keys { - tab.AddRow(key) - } - - return tab.Write(globalOptions.stdout) -} - -// testKeyNewPassword is used to set a new password during integration testing. -var testKeyNewPassword string - -func getNewPassword(gopts GlobalOptions) (string, error) { - if testKeyNewPassword != "" { - return testKeyNewPassword, nil - } - - if newPasswordFile != "" { - return loadPasswordFromFile(newPasswordFile) - } - - // Since we already have an open repository, temporary remove the password - // to prompt the user for the passwd. - newopts := gopts - newopts.password = "" - - return ReadPasswordTwice(newopts, - "enter new password: ", - "enter password again: ") -} - -func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error { - pw, err := getNewPassword(gopts) - if err != nil { - return err - } - - id, err := repository.AddKey(ctx, repo, pw, keyUsername, keyHostname, repo.Key()) - if err != nil { - return errors.Fatalf("creating new key failed: %v\n", err) - } - - err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw) - if err != nil { - return err - } - - Verbosef("saved new key as %s\n", id) - - return nil -} - -func deleteKey(ctx context.Context, repo *repository.Repository, id restic.ID) error { - if id == repo.KeyID() { - return errors.Fatal("refusing to remove key currently used to access repository") - } - - h := backend.Handle{Type: restic.KeyFile, Name: id.String()} - err := repo.Backend().Remove(ctx, h) - if err != nil { - return err - } - - Verbosef("removed key %v\n", id) - return nil -} - -func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error { - pw, err := getNewPassword(gopts) - if err != nil { - return err - } - - id, err := repository.AddKey(ctx, repo, pw, "", "", repo.Key()) - if err != nil { - return errors.Fatalf("creating new key failed: %v\n", err) - } - oldID := repo.KeyID() - - err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw) - if err != nil { - return err - } - - h := backend.Handle{Type: restic.KeyFile, Name: oldID.String()} - err = repo.Backend().Remove(ctx, h) - if err != nil { - return err - } - - Verbosef("saved new key as %s\n", id) - - return nil -} - -func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error { - // Verify new key to make sure it really works. A broken key can render the - // whole repository inaccessible - err := repo.SearchKey(ctx, pw, 0, key.ID().String()) - if err != nil { - // the key is invalid, try to remove it - h := backend.Handle{Type: restic.KeyFile, Name: key.ID().String()} - _ = repo.Backend().Remove(ctx, h) - return errors.Fatalf("failed to access repository with new key: %v", err) - } - return nil -} - -func runKey(ctx context.Context, gopts GlobalOptions, args []string) error { - if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) { - return errors.Fatal("wrong number of arguments") - } - - repo, err := OpenRepository(ctx, gopts) - if err != nil { - return err - } - - switch args[0] { - case "list": - if !gopts.NoLock { - var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - } - - return listKeys(ctx, repo, gopts) - case "add": - lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - - return addKey(ctx, repo, gopts) - case "remove": - lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - - id, err := restic.Find(ctx, repo, restic.KeyFile, args[1]) - if err != nil { - return err - } - - return deleteKey(ctx, repo, id) - case "passwd": - lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - - return changePassword(ctx, repo, gopts) - } - - return nil -} - -func loadPasswordFromFile(pwdFile string) (string, error) { - s, err := os.ReadFile(pwdFile) - if os.IsNotExist(err) { - return "", errors.Fatalf("%s does not exist", pwdFile) - } - return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile") } diff --git a/cmd/restic/cmd_key_add.go b/cmd/restic/cmd_key_add.go new file mode 100644 index 000000000..43a38f4eb --- /dev/null +++ b/cmd/restic/cmd_key_add.go @@ -0,0 +1,128 @@ +package main + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" + "github.com/spf13/cobra" +) + +var cmdKeyAdd = &cobra.Command{ + Use: "add", + Short: "Add a new key (password) to the repository; returns the new key ID", + Long: ` +The "add" sub-command creates a new key and validates the key. Returns the new key ID. + +EXIT STATUS +=========== + +Exit status is 0 if the command is successful, and non-zero if there was any error. + `, + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + return runKeyAdd(cmd.Context(), globalOptions, keyAddOpts, args) + }, +} + +type KeyAddOptions struct { + NewPasswordFile string + Username string + Hostname string +} + +var keyAddOpts KeyAddOptions + +func init() { + cmdKey.AddCommand(cmdKeyAdd) + + flags := cmdKeyAdd.Flags() + flags.StringVarP(&keyAddOpts.NewPasswordFile, "new-password-file", "", "", "`file` from which to read the new password") + flags.StringVarP(&keyAddOpts.Username, "user", "", "", "the username for new key") + flags.StringVarP(&keyAddOpts.Hostname, "host", "", "", "the hostname for new key") +} + +func runKeyAdd(ctx context.Context, gopts GlobalOptions, opts KeyAddOptions, args []string) error { + if len(args) > 0 { + return fmt.Errorf("the key add command expects no arguments, only options - please see `restic help key add` for usage and flags") + } + + repo, err := OpenRepository(ctx, gopts) + if err != nil { + return err + } + + lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) + defer unlockRepo(lock) + if err != nil { + return err + } + + return addKey(ctx, repo, gopts, opts) +} + +func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyAddOptions) error { + pw, err := getNewPassword(gopts, opts.NewPasswordFile) + if err != nil { + return err + } + + id, err := repository.AddKey(ctx, repo, pw, opts.Username, opts.Hostname, repo.Key()) + if err != nil { + return errors.Fatalf("creating new key failed: %v\n", err) + } + + err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw) + if err != nil { + return err + } + + Verbosef("saved new key with ID %s\n", id.ID()) + + return nil +} + +// testKeyNewPassword is used to set a new password during integration testing. +var testKeyNewPassword string + +func getNewPassword(gopts GlobalOptions, newPasswordFile string) (string, error) { + if testKeyNewPassword != "" { + return testKeyNewPassword, nil + } + + if newPasswordFile != "" { + return loadPasswordFromFile(newPasswordFile) + } + + // Since we already have an open repository, temporary remove the password + // to prompt the user for the passwd. + newopts := gopts + newopts.password = "" + + return ReadPasswordTwice(newopts, + "enter new password: ", + "enter password again: ") +} + +func loadPasswordFromFile(pwdFile string) (string, error) { + s, err := os.ReadFile(pwdFile) + if os.IsNotExist(err) { + return "", errors.Fatalf("%s does not exist", pwdFile) + } + return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile") +} + +func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error { + // Verify new key to make sure it really works. A broken key can render the + // whole repository inaccessible + err := repo.SearchKey(ctx, pw, 0, key.ID().String()) + if err != nil { + // the key is invalid, try to remove it + _ = repository.RemoveKey(ctx, repo, key.ID()) + return errors.Fatalf("failed to access repository with new key: %v", err) + } + return nil +} diff --git a/cmd/restic/cmd_key_integration_test.go b/cmd/restic/cmd_key_integration_test.go index f68799dde..16cc1bdad 100644 --- a/cmd/restic/cmd_key_integration_test.go +++ b/cmd/restic/cmd_key_integration_test.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "regexp" + "strings" "testing" "github.com/restic/restic/internal/backend" @@ -13,7 +14,7 @@ import ( func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string { buf, err := withCaptureStdout(func() error { - return runKey(context.TODO(), gopts, []string{"list"}) + return runKeyList(context.TODO(), gopts, []string{}) }) rtest.OK(t, err) @@ -36,21 +37,20 @@ func testRunKeyAddNewKey(t testing.TB, newPassword string, gopts GlobalOptions) testKeyNewPassword = "" }() - rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"})) + rtest.OK(t, runKeyAdd(context.TODO(), gopts, KeyAddOptions{}, []string{})) } func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) { testKeyNewPassword = "john's geheimnis" defer func() { testKeyNewPassword = "" - keyUsername = "" - keyHostname = "" }() - rtest.OK(t, cmdKey.Flags().Parse([]string{"--user=john", "--host=example.com"})) - t.Log("adding key for john@example.com") - rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"})) + rtest.OK(t, runKeyAdd(context.TODO(), gopts, KeyAddOptions{ + Username: "john", + Hostname: "example.com", + }, []string{})) repo, err := OpenRepository(context.TODO(), gopts) rtest.OK(t, err) @@ -67,13 +67,13 @@ func testRunKeyPasswd(t testing.TB, newPassword string, gopts GlobalOptions) { testKeyNewPassword = "" }() - rtest.OK(t, runKey(context.TODO(), gopts, []string{"passwd"})) + rtest.OK(t, runKeyPasswd(context.TODO(), gopts, KeyPasswdOptions{}, []string{})) } func testRunKeyRemove(t testing.TB, gopts GlobalOptions, IDs []string) { t.Logf("remove %d keys: %q\n", len(IDs), IDs) for _, id := range IDs { - rtest.OK(t, runKey(context.TODO(), gopts, []string{"remove", id})) + rtest.OK(t, runKeyRemove(context.TODO(), gopts, []string{id})) } } @@ -103,7 +103,7 @@ func TestKeyAddRemove(t *testing.T) { env.gopts.password = passwordList[len(passwordList)-1] t.Logf("testing access with last password %q\n", env.gopts.password) - rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"})) + rtest.OK(t, runKeyList(context.TODO(), env.gopts, []string{})) testRunCheck(t, env.gopts) testRunKeyAddNewKeyUserHost(t, env.gopts) @@ -131,15 +131,45 @@ func TestKeyProblems(t *testing.T) { testKeyNewPassword = "" }() - err := runKey(context.TODO(), env.gopts, []string{"passwd"}) + err := runKeyPasswd(context.TODO(), env.gopts, KeyPasswdOptions{}, []string{}) t.Log(err) rtest.Assert(t, err != nil, "expected passwd change to fail") - err = runKey(context.TODO(), env.gopts, []string{"add"}) + err = runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{}, []string{}) t.Log(err) rtest.Assert(t, err != nil, "expected key adding to fail") t.Logf("testing access with initial password %q\n", env.gopts.password) - rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"})) + rtest.OK(t, runKeyList(context.TODO(), env.gopts, []string{})) testRunCheck(t, env.gopts) } + +func TestKeyCommandInvalidArguments(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { + return &emptySaveBackend{r}, nil + } + + err := runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{}, []string{"johndoe"}) + t.Log(err) + rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key add: %v", err) + + err = runKeyPasswd(context.TODO(), env.gopts, KeyPasswdOptions{}, []string{"johndoe"}) + t.Log(err) + rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key passwd: %v", err) + + err = runKeyList(context.TODO(), env.gopts, []string{"johndoe"}) + t.Log(err) + rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key list: %v", err) + + err = runKeyRemove(context.TODO(), env.gopts, []string{}) + t.Log(err) + rtest.Assert(t, err != nil && strings.Contains(err.Error(), "one argument"), "unexpected error for key remove: %v", err) + + err = runKeyRemove(context.TODO(), env.gopts, []string{"john", "doe"}) + t.Log(err) + rtest.Assert(t, err != nil && strings.Contains(err.Error(), "one argument"), "unexpected error for key remove: %v", err) +} diff --git a/cmd/restic/cmd_key_list.go b/cmd/restic/cmd_key_list.go new file mode 100644 index 000000000..2b3574281 --- /dev/null +++ b/cmd/restic/cmd_key_list.go @@ -0,0 +1,112 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "sync" + + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/table" + "github.com/spf13/cobra" +) + +var cmdKeyList = &cobra.Command{ + Use: "list", + Short: "List keys (passwords)", + Long: ` +The "list" sub-command lists all the keys (passwords) associated with the repository. +Returns the key ID, username, hostname, created time and if it's the current key being +used to access the repository. + +EXIT STATUS +=========== + +Exit status is 0 if the command is successful, and non-zero if there was any error. + `, + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + return runKeyList(cmd.Context(), globalOptions, args) + }, +} + +func init() { + cmdKey.AddCommand(cmdKeyList) +} + +func runKeyList(ctx context.Context, gopts GlobalOptions, args []string) error { + if len(args) > 0 { + return fmt.Errorf("the key list command expects no arguments, only options - please see `restic help key list` for usage and flags") + } + + repo, err := OpenRepository(ctx, gopts) + if err != nil { + return err + } + + if !gopts.NoLock { + var lock *restic.Lock + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) + defer unlockRepo(lock) + if err != nil { + return err + } + } + + return listKeys(ctx, repo, gopts) +} + +func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions) error { + type keyInfo struct { + Current bool `json:"current"` + ID string `json:"id"` + UserName string `json:"userName"` + HostName string `json:"hostName"` + Created string `json:"created"` + } + + var m sync.Mutex + var keys []keyInfo + + err := restic.ParallelList(ctx, s, restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, _ int64) error { + k, err := repository.LoadKey(ctx, s, id) + if err != nil { + Warnf("LoadKey() failed: %v\n", err) + return nil + } + + key := keyInfo{ + Current: id == s.KeyID(), + ID: id.Str(), + UserName: k.Username, + HostName: k.Hostname, + Created: k.Created.Local().Format(TimeFormat), + } + + m.Lock() + defer m.Unlock() + keys = append(keys, key) + return nil + }) + + if err != nil { + return err + } + + if gopts.JSON { + return json.NewEncoder(globalOptions.stdout).Encode(keys) + } + + tab := table.New() + tab.AddColumn(" ID", "{{if .Current}}*{{else}} {{end}}{{ .ID }}") + tab.AddColumn("User", "{{ .UserName }}") + tab.AddColumn("Host", "{{ .HostName }}") + tab.AddColumn("Created", "{{ .Created }}") + + for _, key := range keys { + tab.AddRow(key) + } + + return tab.Write(globalOptions.stdout) +} diff --git a/cmd/restic/cmd_key_passwd.go b/cmd/restic/cmd_key_passwd.go new file mode 100644 index 000000000..cb916274c --- /dev/null +++ b/cmd/restic/cmd_key_passwd.go @@ -0,0 +1,89 @@ +package main + +import ( + "context" + "fmt" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" + "github.com/spf13/cobra" +) + +var cmdKeyPasswd = &cobra.Command{ + Use: "passwd", + Short: "Change key (password); creates a new key ID and removes the old key ID, returns new key ID", + Long: ` +The "passwd" sub-command creates a new key, validates the key and remove the old key ID. +Returns the new key ID. + +EXIT STATUS +=========== + +Exit status is 0 if the command is successful, and non-zero if there was any error. + `, + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + return runKeyPasswd(cmd.Context(), globalOptions, keyPasswdOpts, args) + }, +} + +type KeyPasswdOptions struct { + KeyAddOptions +} + +var keyPasswdOpts KeyPasswdOptions + +func init() { + cmdKey.AddCommand(cmdKeyPasswd) + + flags := cmdKeyPasswd.Flags() + flags.StringVarP(&keyPasswdOpts.NewPasswordFile, "new-password-file", "", "", "`file` from which to read the new password") + flags.StringVarP(&keyPasswdOpts.Username, "user", "", "", "the username for new key") + flags.StringVarP(&keyPasswdOpts.Hostname, "host", "", "", "the hostname for new key") +} + +func runKeyPasswd(ctx context.Context, gopts GlobalOptions, opts KeyPasswdOptions, args []string) error { + if len(args) > 0 { + return fmt.Errorf("the key passwd command expects no arguments, only options - please see `restic help key passwd` for usage and flags") + } + + repo, err := OpenRepository(ctx, gopts) + if err != nil { + return err + } + + lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) + defer unlockRepo(lock) + if err != nil { + return err + } + + return changePassword(ctx, repo, gopts, opts) +} + +func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyPasswdOptions) error { + pw, err := getNewPassword(gopts, opts.NewPasswordFile) + if err != nil { + return err + } + + id, err := repository.AddKey(ctx, repo, pw, "", "", repo.Key()) + if err != nil { + return errors.Fatalf("creating new key failed: %v\n", err) + } + oldID := repo.KeyID() + + err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw) + if err != nil { + return err + } + + err = repository.RemoveKey(ctx, repo, oldID) + if err != nil { + return err + } + + Verbosef("saved new key as %s\n", id) + + return nil +} diff --git a/cmd/restic/cmd_key_remove.go b/cmd/restic/cmd_key_remove.go new file mode 100644 index 000000000..c8e303ffc --- /dev/null +++ b/cmd/restic/cmd_key_remove.go @@ -0,0 +1,73 @@ +package main + +import ( + "context" + "fmt" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + "github.com/spf13/cobra" +) + +var cmdKeyRemove = &cobra.Command{ + Use: "remove [ID]", + Short: "Remove key ID (password) from the repository.", + Long: ` +The "remove" sub-command removes the selected key ID. The "remove" command does not allow +removing the current key being used to access the repository. + +EXIT STATUS +=========== + +Exit status is 0 if the command is successful, and non-zero if there was any error. + `, + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + return runKeyRemove(cmd.Context(), globalOptions, args) + }, +} + +func init() { + cmdKey.AddCommand(cmdKeyRemove) +} + +func runKeyRemove(ctx context.Context, gopts GlobalOptions, args []string) error { + if len(args) != 1 { + return fmt.Errorf("key remove expects one argument as the key id") + } + + repo, err := OpenRepository(ctx, gopts) + if err != nil { + return err + } + + lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) + defer unlockRepo(lock) + if err != nil { + return err + } + + idPrefix := args[0] + + return deleteKey(ctx, repo, idPrefix) +} + +func deleteKey(ctx context.Context, repo *repository.Repository, idPrefix string) error { + id, err := restic.Find(ctx, repo, restic.KeyFile, idPrefix) + if err != nil { + return err + } + + if id == repo.KeyID() { + return errors.Fatal("refusing to remove key currently used to access repository") + } + + err = repository.RemoveKey(ctx, repo, id) + if err != nil { + return err + } + + Verbosef("removed key %v\n", id) + return nil +} diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go index 38f8b094a..becad7f0d 100644 --- a/cmd/restic/cmd_list.go +++ b/cmd/restic/cmd_list.go @@ -23,7 +23,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runList(cmd.Context(), cmd, globalOptions, args) + return runList(cmd.Context(), globalOptions, args) }, } @@ -31,9 +31,9 @@ func init() { cmdRoot.AddCommand(cmdList) } -func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args []string) error { +func runList(ctx context.Context, gopts GlobalOptions, args []string) error { if len(args) != 1 { - return errors.Fatal("type not specified, usage: " + cmd.Use) + return errors.Fatal("type not specified") } repo, err := OpenRepository(ctx, gopts) @@ -63,7 +63,7 @@ func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args case "locks": t = restic.LockFile case "blobs": - return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { + return index.ForAllIndexes(ctx, repo, repo, func(_ restic.ID, idx *index.Index, _ bool, err error) error { if err != nil { return err } @@ -76,7 +76,7 @@ func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args return errors.Fatal("invalid type") } - return repo.List(ctx, t, func(id restic.ID, size int64) error { + return repo.List(ctx, t, func(id restic.ID, _ int64) error { Printf("%s\n", id) return nil }) diff --git a/cmd/restic/cmd_list_integration_test.go b/cmd/restic/cmd_list_integration_test.go index 4140a3ea8..ef2b8bf8f 100644 --- a/cmd/restic/cmd_list_integration_test.go +++ b/cmd/restic/cmd_list_integration_test.go @@ -12,7 +12,7 @@ import ( func testRunList(t testing.TB, tpe string, opts GlobalOptions) restic.IDs { buf, err := withCaptureStdout(func() error { - return runList(context.TODO(), cmdList, opts, []string{tpe}) + return runList(context.TODO(), opts, []string{tpe}) }) rtest.OK(t, err) return parseIDsFromReader(t, buf) diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 83a03559d..b0246625e 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -3,6 +3,8 @@ package main import ( "context" "encoding/json" + "fmt" + "io" "os" "strings" "time" @@ -51,6 +53,7 @@ type LsOptions struct { restic.SnapshotFilter Recursive bool HumanReadable bool + Ncdu bool } var lsOptions LsOptions @@ -63,16 +66,49 @@ func init() { flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode") flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories") flags.BoolVar(&lsOptions.HumanReadable, "human-readable", false, "print sizes in human readable format") + flags.BoolVar(&lsOptions.Ncdu, "ncdu", false, "output NCDU export format (pipe into 'ncdu -f -')") } -type lsSnapshot struct { - *restic.Snapshot - ID *restic.ID `json:"id"` - ShortID string `json:"short_id"` - StructType string `json:"struct_type"` // "snapshot" +type lsPrinter interface { + Snapshot(sn *restic.Snapshot) + Node(path string, node *restic.Node) + LeaveDir(path string) + Close() +} + +type jsonLsPrinter struct { + enc *json.Encoder +} + +func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) { + type lsSnapshot struct { + *restic.Snapshot + ID *restic.ID `json:"id"` + ShortID string `json:"short_id"` + MessageType string `json:"message_type"` // "snapshot" + StructType string `json:"struct_type"` // "snapshot", deprecated + } + + err := p.enc.Encode(lsSnapshot{ + Snapshot: sn, + ID: sn.ID(), + ShortID: sn.ID().Str(), + MessageType: "snapshot", + StructType: "snapshot", + }) + if err != nil { + Warnf("JSON encode failed: %v\n", err) + } } // Print node in our custom JSON format, followed by a newline. +func (p *jsonLsPrinter) Node(path string, node *restic.Node) { + err := lsNodeJSON(p.enc, path, node) + if err != nil { + Warnf("JSON encode failed: %v\n", err) + } +} + func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { n := &struct { Name string `json:"name"` @@ -87,7 +123,8 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { AccessTime time.Time `json:"atime,omitempty"` ChangeTime time.Time `json:"ctime,omitempty"` Inode uint64 `json:"inode,omitempty"` - StructType string `json:"struct_type"` // "node" + MessageType string `json:"message_type"` // "node" + StructType string `json:"struct_type"` // "node", deprecated size uint64 // Target for Size pointer. }{ @@ -103,6 +140,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { AccessTime: node.AccessTime, ChangeTime: node.ChangeTime, Inode: node.Inode, + MessageType: "node", StructType: "node", } // Always print size for regular files, even when empty, @@ -114,10 +152,117 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { return enc.Encode(n) } +func (p *jsonLsPrinter) LeaveDir(_ string) {} +func (p *jsonLsPrinter) Close() {} + +type ncduLsPrinter struct { + out io.Writer + depth int +} + +// lsSnapshotNcdu prints a restic snapshot in Ncdu save format. +// It opens the JSON list. Nodes are added with lsNodeNcdu and the list is closed by lsCloseNcdu. +// Format documentation: https://dev.yorhel.nl/ncdu/jsonfmt +func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) { + const NcduMajorVer = 1 + const NcduMinorVer = 2 + + snapshotBytes, err := json.Marshal(sn) + if err != nil { + Warnf("JSON encode failed: %v\n", err) + } + p.depth++ + fmt.Fprintf(p.out, "[%d, %d, %s", NcduMajorVer, NcduMinorVer, string(snapshotBytes)) +} + +func lsNcduNode(_ string, node *restic.Node) ([]byte, error) { + type NcduNode struct { + Name string `json:"name"` + Asize uint64 `json:"asize"` + Dsize uint64 `json:"dsize"` + Dev uint64 `json:"dev"` + Ino uint64 `json:"ino"` + NLink uint64 `json:"nlink"` + NotReg bool `json:"notreg"` + UID uint32 `json:"uid"` + GID uint32 `json:"gid"` + Mode uint16 `json:"mode"` + Mtime int64 `json:"mtime"` + } + + outNode := NcduNode{ + Name: node.Name, + Asize: node.Size, + Dsize: node.Size, + Dev: node.DeviceID, + Ino: node.Inode, + NLink: node.Links, + NotReg: node.Type != "dir" && node.Type != "file", + UID: node.UID, + GID: node.GID, + Mode: uint16(node.Mode & os.ModePerm), + Mtime: node.ModTime.Unix(), + } + // bits according to inode(7) manpage + if node.Mode&os.ModeSetuid != 0 { + outNode.Mode |= 0o4000 + } + if node.Mode&os.ModeSetgid != 0 { + outNode.Mode |= 0o2000 + } + if node.Mode&os.ModeSticky != 0 { + outNode.Mode |= 0o1000 + } + + return json.Marshal(outNode) +} + +func (p *ncduLsPrinter) Node(path string, node *restic.Node) { + out, err := lsNcduNode(path, node) + if err != nil { + Warnf("JSON encode failed: %v\n", err) + } + + if node.Type == "dir" { + fmt.Fprintf(p.out, ",\n%s[\n%s%s", strings.Repeat(" ", p.depth), strings.Repeat(" ", p.depth+1), string(out)) + p.depth++ + } else { + fmt.Fprintf(p.out, ",\n%s%s", strings.Repeat(" ", p.depth), string(out)) + } +} + +func (p *ncduLsPrinter) LeaveDir(_ string) { + p.depth-- + fmt.Fprintf(p.out, "\n%s]", strings.Repeat(" ", p.depth)) +} + +func (p *ncduLsPrinter) Close() { + fmt.Fprint(p.out, "\n]\n") +} + +type textLsPrinter struct { + dirs []string + ListLong bool + HumanReadable bool +} + +func (p *textLsPrinter) Snapshot(sn *restic.Snapshot) { + Verbosef("%v filtered by %v:\n", sn, p.dirs) +} +func (p *textLsPrinter) Node(path string, node *restic.Node) { + Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable)) +} + +func (p *textLsPrinter) LeaveDir(_ string) {} +func (p *textLsPrinter) Close() {} + func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error { if len(args) == 0 { return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'") } + if opts.Ncdu && gopts.JSON { + return errors.Fatal("only either '--json' or '--ncdu' can be specified") + } // extract any specific directories to walk var dirs []string @@ -179,38 +324,21 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return err } - var ( - printSnapshot func(sn *restic.Snapshot) - printNode func(path string, node *restic.Node) - ) + var printer lsPrinter if gopts.JSON { - enc := json.NewEncoder(globalOptions.stdout) - - printSnapshot = func(sn *restic.Snapshot) { - err := enc.Encode(lsSnapshot{ - Snapshot: sn, - ID: sn.ID(), - ShortID: sn.ID().Str(), - StructType: "snapshot", - }) - if err != nil { - Warnf("JSON encode failed: %v\n", err) - } + printer = &jsonLsPrinter{ + enc: json.NewEncoder(globalOptions.stdout), } - - printNode = func(path string, node *restic.Node) { - err := lsNodeJSON(enc, path, node) - if err != nil { - Warnf("JSON encode failed: %v\n", err) - } + } else if opts.Ncdu { + printer = &ncduLsPrinter{ + out: globalOptions.stdout, } } else { - printSnapshot = func(sn *restic.Snapshot) { - Verbosef("%v filtered by %v:\n", sn, dirs) - } - printNode = func(path string, node *restic.Node) { - Printf("%s\n", formatNode(path, node, lsOptions.ListLong, lsOptions.HumanReadable)) + printer = &textLsPrinter{ + dirs: dirs, + ListLong: opts.ListLong, + HumanReadable: opts.HumanReadable, } } @@ -228,44 +356,55 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return err } - printSnapshot(sn) + printer.Snapshot(sn) - err = walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { + processNode := func(_ restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { - return false, err + return err } if node == nil { - return false, nil + return nil } if withinDir(nodepath) { // if we're within a dir, print the node - printNode(nodepath, node) + printer.Node(nodepath, node) // if recursive listing is requested, signal the walker that it // should continue walking recursively if opts.Recursive { - return false, nil + return nil } } // if there's an upcoming match deeper in the tree (but we're not // there yet), signal the walker to descend into any subdirs if approachingMatchingTree(nodepath) { - return false, nil + return nil } // otherwise, signal the walker to not walk recursively into any // subdirs if node.Type == "dir" { - return false, walker.ErrSkipNode + return walker.ErrSkipNode } - return false, nil + return nil + } + + err = walker.Walk(ctx, repo, *sn.Tree, walker.WalkVisitor{ + ProcessNode: processNode, + LeaveDir: func(path string) { + // the root path `/` has no corresponding node and is thus also skipped by processNode + if withinDir(path) && path != "/" { + printer.LeaveDir(path) + } + }, }) if err != nil { return err } + printer.Close() return nil } diff --git a/cmd/restic/cmd_ls_integration_test.go b/cmd/restic/cmd_ls_integration_test.go index 39bf9c3b0..1b3c964e4 100644 --- a/cmd/restic/cmd_ls_integration_test.go +++ b/cmd/restic/cmd_ls_integration_test.go @@ -2,18 +2,46 @@ package main import ( "context" + "encoding/json" + "path/filepath" "strings" "testing" rtest "github.com/restic/restic/internal/test" ) -func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string { +func testRunLsWithOpts(t testing.TB, gopts GlobalOptions, opts LsOptions, args []string) []byte { buf, err := withCaptureStdout(func() error { gopts.Quiet = true - opts := LsOptions{} - return runLs(context.TODO(), opts, gopts, []string{snapshotID}) + return runLs(context.TODO(), opts, gopts, args) }) rtest.OK(t, err) - return strings.Split(buf.String(), "\n") + return buf.Bytes() +} + +func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string { + out := testRunLsWithOpts(t, gopts, LsOptions{}, []string{snapshotID}) + return strings.Split(string(out), "\n") +} + +func assertIsValidJSON(t *testing.T, data []byte) { + // Sanity check: output must be valid JSON. + var v interface{} + err := json.Unmarshal(data, &v) + rtest.OK(t, err) +} + +func TestRunLsNcdu(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + opts := BackupOptions{} + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts) + + ncdu := testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest"}) + assertIsValidJSON(t, ncdu) + + ncdu = testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest", "/testdata"}) + assertIsValidJSON(t, ncdu) } diff --git a/cmd/restic/cmd_ls_test.go b/cmd/restic/cmd_ls_test.go index 8a4fa51ee..828b2920e 100644 --- a/cmd/restic/cmd_ls_test.go +++ b/cmd/restic/cmd_ls_test.go @@ -11,78 +11,94 @@ import ( rtest "github.com/restic/restic/internal/test" ) +type lsTestNode struct { + path string + restic.Node +} + +var lsTestNodes = []lsTestNode{ + // Mode is omitted when zero. + // Permissions, by convention is "-" per mode bit + { + path: "/bar/baz", + Node: restic.Node{ + Name: "baz", + Type: "file", + Size: 12345, + UID: 10000000, + GID: 20000000, + + User: "nobody", + Group: "nobodies", + Links: 1, + }, + }, + + // Even empty files get an explicit size. + { + path: "/foo/empty", + Node: restic.Node{ + Name: "empty", + Type: "file", + Size: 0, + UID: 1001, + GID: 1001, + + User: "not printed", + Group: "not printed", + Links: 0xF00, + }, + }, + + // Non-regular files do not get a size. + // Mode is printed in decimal, including the type bits. + { + path: "/foo/link", + Node: restic.Node{ + Name: "link", + Type: "symlink", + Mode: os.ModeSymlink | 0777, + LinkTarget: "not printed", + }, + }, + + { + path: "/some/directory", + Node: restic.Node{ + Name: "directory", + Type: "dir", + Mode: os.ModeDir | 0755, + ModTime: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC), + AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC), + ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC), + }, + }, + + // Test encoding of setuid/setgid/sticky bit + { + path: "/some/sticky", + Node: restic.Node{ + Name: "sticky", + Type: "dir", + Mode: os.ModeDir | 0755 | os.ModeSetuid | os.ModeSetgid | os.ModeSticky, + }, + }, +} + func TestLsNodeJSON(t *testing.T) { - for _, c := range []struct { - path string - restic.Node - expect string - }{ - // Mode is omitted when zero. - // Permissions, by convention is "-" per mode bit - { - path: "/bar/baz", - Node: restic.Node{ - Name: "baz", - Type: "file", - Size: 12345, - UID: 10000000, - GID: 20000000, - - User: "nobody", - Group: "nobodies", - Links: 1, - }, - expect: `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, - }, - - // Even empty files get an explicit size. - { - path: "/foo/empty", - Node: restic.Node{ - Name: "empty", - Type: "file", - Size: 0, - UID: 1001, - GID: 1001, - - User: "not printed", - Group: "not printed", - Links: 0xF00, - }, - expect: `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, - }, - - // Non-regular files do not get a size. - // Mode is printed in decimal, including the type bits. - { - path: "/foo/link", - Node: restic.Node{ - Name: "link", - Type: "symlink", - Mode: os.ModeSymlink | 0777, - LinkTarget: "not printed", - }, - expect: `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, - }, - - { - path: "/some/directory", - Node: restic.Node{ - Name: "directory", - Type: "dir", - Mode: os.ModeDir | 0755, - ModTime: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC), - AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC), - ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC), - }, - expect: `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`, - }, + for i, expect := range []string{ + `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","message_type":"node","struct_type":"node"}`, + `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","message_type":"node","struct_type":"node"}`, + `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","message_type":"node","struct_type":"node"}`, + `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","message_type":"node","struct_type":"node"}`, + `{"name":"sticky","type":"dir","path":"/some/sticky","uid":0,"gid":0,"mode":2161115629,"permissions":"dugtrwxr-xr-x","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","message_type":"node","struct_type":"node"}`, } { + c := lsTestNodes[i] buf := new(bytes.Buffer) enc := json.NewEncoder(buf) err := lsNodeJSON(enc, c.path, &c.Node) rtest.OK(t, err) - rtest.Equals(t, c.expect+"\n", buf.String()) + rtest.Equals(t, expect+"\n", buf.String()) // Sanity check: output must be valid JSON. var v interface{} @@ -90,3 +106,54 @@ func TestLsNodeJSON(t *testing.T) { rtest.OK(t, err) } } + +func TestLsNcduNode(t *testing.T) { + for i, expect := range []string{ + `{"name":"baz","asize":12345,"dsize":12345,"dev":0,"ino":0,"nlink":1,"notreg":false,"uid":10000000,"gid":20000000,"mode":0,"mtime":-62135596800}`, + `{"name":"empty","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":3840,"notreg":false,"uid":1001,"gid":1001,"mode":0,"mtime":-62135596800}`, + `{"name":"link","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":true,"uid":0,"gid":0,"mode":511,"mtime":-62135596800}`, + `{"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":493,"mtime":1577934245}`, + `{"name":"sticky","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":4077,"mtime":-62135596800}`, + } { + c := lsTestNodes[i] + out, err := lsNcduNode(c.path, &c.Node) + rtest.OK(t, err) + rtest.Equals(t, expect, string(out)) + + // Sanity check: output must be valid JSON. + var v interface{} + err = json.Unmarshal(out, &v) + rtest.OK(t, err) + } +} + +func TestLsNcdu(t *testing.T) { + var buf bytes.Buffer + printer := &ncduLsPrinter{ + out: &buf, + } + + printer.Snapshot(&restic.Snapshot{ + Hostname: "host", + Paths: []string{"/example"}, + }) + printer.Node("/directory", &restic.Node{ + Type: "dir", + Name: "directory", + }) + printer.Node("/directory/data", &restic.Node{ + Type: "file", + Name: "data", + Size: 42, + }) + printer.LeaveDir("/directory") + printer.Close() + + rtest.Equals(t, `[1, 2, {"time":"0001-01-01T00:00:00Z","tree":null,"paths":["/example"],"hostname":"host"}, + [ + {"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800}, + {"name":"data","asize":42,"dsize":42,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800} + ] +] +`, buf.String()) +} diff --git a/cmd/restic/cmd_mount_integration_test.go b/cmd/restic/cmd_mount_integration_test.go index 1b069d582..d2025a395 100644 --- a/cmd/restic/cmd_mount_integration_test.go +++ b/cmd/restic/cmd_mount_integration_test.go @@ -12,7 +12,6 @@ import ( "testing" "time" - "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -160,11 +159,6 @@ func TestMount(t *testing.T) { t.Skip("Skipping fuse tests") } - debugEnabled := debug.TestLogToStderr(t) - if debugEnabled { - defer debug.TestDisableLog(t) - } - env, cleanup := withTestEnvironment(t) // must list snapshots more than once env.gopts.backendTestHook = nil diff --git a/cmd/restic/cmd_options.go b/cmd/restic/cmd_options.go index 471319dfb..85e062220 100644 --- a/cmd/restic/cmd_options.go +++ b/cmd/restic/cmd_options.go @@ -21,7 +21,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, Hidden: true, DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, _ []string) { fmt.Printf("All Extended Options:\n") var maxLen int for _, opt := range options.List() { diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index 739a450df..1b9352ea7 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -15,6 +15,7 @@ import ( "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui" + "github.com/restic/restic/internal/ui/progress" "github.com/spf13/cobra" ) @@ -36,7 +37,7 @@ EXIT STATUS Exit status is 0 if the command was successful, and non-zero if there was any error. `, DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { return runPrune(cmd.Context(), pruneOptions, globalOptions) }, } @@ -66,10 +67,10 @@ func init() { f := cmdPrune.Flags() f.BoolVarP(&pruneOptions.DryRun, "dry-run", "n", false, "do not modify the repository, just print what would be done") f.StringVarP(&pruneOptions.UnsafeNoSpaceRecovery, "unsafe-recover-no-free-space", "", "", "UNSAFE, READ THE DOCUMENTATION BEFORE USING! Try to recover a repository stuck with no free space. Do not use without trying out 'prune --max-repack-size 0' first.") - addPruneOptions(cmdPrune) + addPruneOptions(cmdPrune, &pruneOptions) } -func addPruneOptions(c *cobra.Command) { +func addPruneOptions(c *cobra.Command, pruneOptions *PruneOptions) { f := c.Flags() f.StringVar(&pruneOptions.MaxUnused, "max-unused", "5%", "tolerate given `limit` of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited')") f.StringVar(&pruneOptions.MaxRepackSize, "max-repack-size", "", "maximum `size` to repack (allowed suffixes: k/K, m/M, g/G, t/T)") @@ -100,7 +101,7 @@ func verifyPruneOptions(opts *PruneOptions) error { // parse MaxUnused either as unlimited, a percentage, or an absolute number of bytes switch { case maxUnused == "unlimited": - opts.maxUnusedBytes = func(used uint64) uint64 { + opts.maxUnusedBytes = func(_ uint64) uint64 { return math.MaxUint64 } @@ -129,7 +130,7 @@ func verifyPruneOptions(opts *PruneOptions) error { return errors.Fatalf("invalid number of bytes %q for --max-unused: %v", opts.MaxUnused, err) } - opts.maxUnusedBytes = func(used uint64) uint64 { + opts.maxUnusedBytes = func(_ uint64) uint64 { return uint64(size) } } @@ -766,7 +767,7 @@ func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo r return errors.Fatalf("%s", err) } } else if len(plan.ignorePacks) != 0 { - err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil) + err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil, false) if err != nil { return errors.Fatalf("%s", err) } @@ -778,7 +779,7 @@ func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo r } if opts.unsafeRecovery { - _, err = writeIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil) + err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil, true) if err != nil { return errors.Fatalf("%s", err) } @@ -788,23 +789,22 @@ func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo r return nil } -func writeIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) (restic.IDSet, error) { +func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs, skipDeletion bool) error { Verbosef("rebuilding index\n") bar := newProgressMax(!gopts.Quiet, 0, "packs processed") - obsoleteIndexes, err := repo.Index().Save(ctx, repo, removePacks, extraObsolete, bar) - bar.Done() - return obsoleteIndexes, err -} - -func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error { - obsoleteIndexes, err := writeIndexFiles(ctx, gopts, repo, removePacks, extraObsolete) - if err != nil { - return err - } - - Verbosef("deleting obsolete index files\n") - return DeleteFilesChecked(ctx, gopts, repo, obsoleteIndexes, restic.IndexFile) + return repo.Index().Save(ctx, repo, removePacks, extraObsolete, restic.MasterIndexSaveOpts{ + SaveProgress: bar, + DeleteProgress: func() *progress.Counter { + return newProgressMax(!gopts.Quiet, 0, "old indexes deleted") + }, + DeleteReport: func(id restic.ID, _ error) { + if gopts.verbosity > 2 { + Verbosef("removed index %v\n", id.String()) + } + }, + SkipDeletion: skipDeletion, + }) } func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (usedBlobs restic.CountedBlobSet, err error) { diff --git a/cmd/restic/cmd_prune_integration_test.go b/cmd/restic/cmd_prune_integration_test.go index 53e27ee10..ebfa7ae4e 100644 --- a/cmd/restic/cmd_prune_integration_test.go +++ b/cmd/restic/cmd_prune_integration_test.go @@ -81,7 +81,10 @@ func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) { DryRun: true, Last: 1, } - return runForget(context.TODO(), opts, gopts, args) + pruneOpts := PruneOptions{ + MaxUnused: "5%", + } + return runForget(context.TODO(), opts, pruneOpts, gopts, args) }) rtest.OK(t, err) diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index ae6aff740..b97a7582b 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -25,7 +25,7 @@ EXIT STATUS Exit status is 0 if the command was successful, and non-zero if there was any error. `, DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { return runRecover(cmd.Context(), globalOptions) }, } @@ -91,7 +91,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error { bar.Done() Verbosef("load snapshots\n") - err = restic.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error { + err = restic.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(_ restic.ID, sn *restic.Snapshot, _ error) error { trees[*sn.Tree] = true return nil }) @@ -158,7 +158,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error { } -func createSnapshot(ctx context.Context, name, hostname string, tags []string, repo restic.Repository, tree *restic.ID) error { +func createSnapshot(ctx context.Context, name, hostname string, tags []string, repo restic.SaverUnpacked, tree *restic.ID) error { sn, err := restic.NewSnapshot([]string{name}, tags, hostname, time.Now()) if err != nil { return errors.Fatalf("unable to save snapshot: %v", err) diff --git a/cmd/restic/cmd_repair_index.go b/cmd/restic/cmd_repair_index.go index c8a94b470..ea36f02f6 100644 --- a/cmd/restic/cmd_repair_index.go +++ b/cmd/restic/cmd_repair_index.go @@ -24,7 +24,7 @@ EXIT STATUS Exit status is 0 if the command was successful, and non-zero if there was any error. `, DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { return runRebuildIndex(cmd.Context(), repairIndexOptions, globalOptions) }, } @@ -78,7 +78,7 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti if opts.ReadAllPacks { // get list of old index files but start with empty index - err := repo.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error { + err := repo.List(ctx, restic.IndexFile, func(id restic.ID, _ int64) error { obsoleteIndexes = append(obsoleteIndexes, id) return nil }) @@ -88,7 +88,7 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti } else { Verbosef("loading indexes...\n") mi := index.NewMasterIndex() - err := index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { + err := index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, _ bool, err error) error { if err != nil { Warnf("removing invalid index %v: %v\n", id, err) obsoleteIndexes = append(obsoleteIndexes, id) @@ -154,7 +154,7 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti } } - err = rebuildIndexFiles(ctx, gopts, repo, removePacks, obsoleteIndexes) + err = rebuildIndexFiles(ctx, gopts, repo, removePacks, obsoleteIndexes, false) if err != nil { return err } diff --git a/cmd/restic/cmd_repair_packs.go b/cmd/restic/cmd_repair_packs.go index 7d1a3a392..521b5859f 100644 --- a/cmd/restic/cmd_repair_packs.go +++ b/cmd/restic/cmd_repair_packs.go @@ -9,8 +9,8 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/termstatus" "github.com/spf13/cobra" - "golang.org/x/sync/errgroup" ) var cmdRepairPacks = &cobra.Command{ @@ -29,7 +29,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runRepairPacks(cmd.Context(), globalOptions, args) + term, cancel := setupTermstatus() + defer cancel() + return runRepairPacks(cmd.Context(), globalOptions, term, args) }, } @@ -37,14 +39,7 @@ func init() { cmdRepair.AddCommand(cmdRepairPacks) } -func runRepairPacks(ctx context.Context, gopts GlobalOptions, args []string) error { - // FIXME discuss and add proper feature flag mechanism - flag, _ := os.LookupEnv("RESTIC_FEATURES") - if flag != "repair-packs-v1" { - return errors.Fatal("This command is experimental and may change/be removed without notice between restic versions. " + - "Set the environment variable 'RESTIC_FEATURES=repair-packs-v1' to enable it.") - } - +func runRepairPacks(ctx context.Context, gopts GlobalOptions, term *termstatus.Terminal, args []string) error { ids := restic.NewIDSet() for _, arg := range args { id, err := restic.ParseID(arg) @@ -68,21 +63,19 @@ func runRepairPacks(ctx context.Context, gopts GlobalOptions, args []string) err return err } - return repairPacks(ctx, gopts, repo, ids) -} - -func repairPacks(ctx context.Context, gopts GlobalOptions, repo *repository.Repository, ids restic.IDSet) error { bar := newIndexProgress(gopts.Quiet, gopts.JSON) - err := repo.LoadIndex(ctx, bar) + err = repo.LoadIndex(ctx, bar) if err != nil { return errors.Fatalf("%s", err) } - Warnf("saving backup copies of pack files in current folder\n") + printer := newTerminalProgressPrinter(gopts.verbosity, term) + + printer.P("saving backup copies of pack files to current folder") for id := range ids { f, err := os.OpenFile("pack-"+id.String(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o666) if err != nil { - return errors.Fatalf("%s", err) + return err } err = repo.Backend().Load(ctx, backend.Handle{Type: restic.PackFile, Name: id.String()}, 0, 0, func(rd io.Reader) error { @@ -94,66 +87,15 @@ func repairPacks(ctx context.Context, gopts GlobalOptions, repo *repository.Repo return err }) if err != nil { - return errors.Fatalf("%s", err) + return err } } - wg, wgCtx := errgroup.WithContext(ctx) - repo.StartPackUploader(wgCtx, wg) - repo.DisableAutoIndexUpdate() - - Warnf("salvaging intact data from specified pack files\n") - bar = newProgressMax(!gopts.Quiet, uint64(len(ids)), "pack files") - defer bar.Done() - - wg.Go(func() error { - // examine all data the indexes have for the pack file - for b := range repo.Index().ListPacks(wgCtx, ids) { - blobs := b.Blobs - if len(blobs) == 0 { - Warnf("no blobs found for pack %v\n", b.PackID) - bar.Add(1) - continue - } - - err = repository.StreamPack(wgCtx, repo.Backend().Load, repo.Key(), b.PackID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { - if err != nil { - // Fallback path - buf, err = repo.LoadBlob(wgCtx, blob.Type, blob.ID, nil) - if err != nil { - Warnf("failed to load blob %v: %v\n", blob.ID, err) - return nil - } - } - id, _, _, err := repo.SaveBlob(wgCtx, blob.Type, buf, restic.ID{}, true) - if !id.Equal(blob.ID) { - panic("pack id mismatch during upload") - } - return err - }) - if err != nil { - return err - } - bar.Add(1) - } - return repo.Flush(wgCtx) - }) - - if err := wg.Wait(); err != nil { - return errors.Fatalf("%s", err) - } - bar.Done() - - // remove salvaged packs from index - err = rebuildIndexFiles(ctx, gopts, repo, ids, nil) + err = repository.RepairPacks(ctx, repo, ids, printer) if err != nil { return errors.Fatalf("%s", err) } - // cleanup - Warnf("removing salvaged pack files\n") - DeleteFiles(ctx, gopts, repo, ids, restic.PackFile) - Warnf("\nUse `restic repair snapshots --forget` to remove the corrupted data blobs from all snapshots\n") return nil } diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index 19e457b1f..cc3d0eb85 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -125,7 +125,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt node.Size = newSize return node }, - RewriteFailedTree: func(nodeID restic.ID, path string, _ error) (restic.ID, error) { + RewriteFailedTree: func(_ restic.ID, path string, _ error) (restic.ID, error) { if path == "/" { Verbosef(" dir %q: not readable\n", path) // remove snapshots with invalid root node diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 6045a5d41..58f257541 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -3,7 +3,6 @@ package main import ( "context" "strings" - "sync" "time" "github.com/restic/restic/internal/debug" @@ -38,31 +37,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - var wg sync.WaitGroup - cancelCtx, cancel := context.WithCancel(ctx) - defer func() { - // shutdown termstatus - cancel() - wg.Wait() - }() - - term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet) - wg.Add(1) - go func() { - defer wg.Done() - term.Run(cancelCtx) - }() - - // allow usage of warnf / verbosef - prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr - defer func() { - globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr - }() - stdioWrapper := ui.NewStdioWrapper(term) - globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr() - - return runRestore(ctx, restoreOptions, globalOptions, term, args) + term, cancel := setupTermstatus() + defer cancel() + return runRestore(cmd.Context(), restoreOptions, globalOptions, term, args) }, } @@ -201,10 +178,13 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, totalErrors++ return nil } + res.Warn = func(message string) { + msg.E("Warning: %s\n", message) + } excludePatterns := filter.ParsePatterns(opts.Exclude) insensitiveExcludePatterns := filter.ParsePatterns(opts.InsensitiveExclude) - selectExcludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { + selectExcludeFilter := func(item string, _ string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { matched, err := filter.List(excludePatterns, item) if err != nil { msg.E("error for exclude pattern: %v", err) @@ -227,7 +207,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, includePatterns := filter.ParsePatterns(opts.Include) insensitiveIncludePatterns := filter.ParsePatterns(opts.InsensitiveInclude) - selectIncludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { + selectIncludeFilter := func(item string, _ string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { matched, childMayMatch, err := filter.ListWithChild(includePatterns, item) if err != nil { msg.E("error for include pattern: %v", err) diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index d55e6137b..62624e75c 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -147,7 +147,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) } } else { - filter = func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { + filter = func(_ context.Context, sn *restic.Snapshot) (restic.ID, error) { return *sn.Tree, nil } } @@ -209,7 +209,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r } if newMetadata != nil && newMetadata.Hostname != "" { - Verbosef("would set time to %s\n", newMetadata.Hostname) + Verbosef("would set hostname to %s\n", newMetadata.Hostname) } return true, nil diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index a3e0cefc7..d3078a419 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -189,7 +189,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args return nil } -func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo restic.Repository, opts StatsOptions, stats *statsContainer) error { +func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo restic.Loader, opts StatsOptions, stats *statsContainer) error { if snapshot.Tree == nil { return fmt.Errorf("snapshot %s has nil tree", snapshot.ID().Str()) } @@ -203,7 +203,9 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest } hardLinkIndex := restorer.NewHardlinkIndex[struct{}]() - err := walker.Walk(ctx, repo, *snapshot.Tree, restic.NewIDSet(), statsWalkTree(repo, opts, stats, hardLinkIndex)) + err := walker.Walk(ctx, repo, *snapshot.Tree, walker.WalkVisitor{ + ProcessNode: statsWalkTree(repo, opts, stats, hardLinkIndex), + }) if err != nil { return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err) } @@ -211,13 +213,13 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest return nil } -func statsWalkTree(repo restic.Repository, opts StatsOptions, stats *statsContainer, hardLinkIndex *restorer.HardlinkIndex[struct{}]) walker.WalkFunc { - return func(parentTreeID restic.ID, npath string, node *restic.Node, nodeErr error) (bool, error) { +func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer, hardLinkIndex *restorer.HardlinkIndex[struct{}]) walker.WalkFunc { + return func(parentTreeID restic.ID, npath string, node *restic.Node, nodeErr error) error { if nodeErr != nil { - return true, nodeErr + return nodeErr } if node == nil { - return true, nil + return nil } if opts.countMode == countModeUniqueFilesByContents || opts.countMode == countModeBlobsPerFile { @@ -247,7 +249,7 @@ func statsWalkTree(repo restic.Repository, opts StatsOptions, stats *statsContai // is always a data blob since we're accessing it via a file's Content array blobSize, found := repo.LookupBlobSize(blobID, restic.DataBlob) if !found { - return true, fmt.Errorf("blob %s not found for tree %s", blobID, parentTreeID) + return fmt.Errorf("blob %s not found for tree %s", blobID, parentTreeID) } // count the blob's size, then add this blob by this @@ -274,11 +276,9 @@ func statsWalkTree(repo restic.Repository, opts StatsOptions, stats *statsContai hardLinkIndex.Add(node.Inode, node.DeviceID, struct{}{}) stats.TotalSize += node.Size } - - return false, nil } - return true, nil + return nil } } @@ -365,9 +365,9 @@ func statsDebug(ctx context.Context, repo restic.Repository) error { return nil } -func statsDebugFileType(ctx context.Context, repo restic.Repository, tpe restic.FileType) (*sizeHistogram, error) { +func statsDebugFileType(ctx context.Context, repo restic.Lister, tpe restic.FileType) (*sizeHistogram, error) { hist := newSizeHistogram(2 * repository.MaxPackSize) - err := repo.List(ctx, tpe, func(id restic.ID, size int64) error { + err := repo.List(ctx, tpe, func(_ restic.ID, size int64) error { hist.Add(uint64(size)) return nil }) diff --git a/cmd/restic/cmd_unlock.go b/cmd/restic/cmd_unlock.go index 7b449d949..6893f3365 100644 --- a/cmd/restic/cmd_unlock.go +++ b/cmd/restic/cmd_unlock.go @@ -19,7 +19,7 @@ EXIT STATUS Exit status is 0 if the command was successful, and non-zero if there was any error. `, DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { return runUnlock(cmd.Context(), unlockOptions, globalOptions) }, } diff --git a/cmd/restic/cmd_version.go b/cmd/restic/cmd_version.go index 73469750f..e3f9b3010 100644 --- a/cmd/restic/cmd_version.go +++ b/cmd/restic/cmd_version.go @@ -21,7 +21,7 @@ EXIT STATUS Exit status is 0 if the command was successful, and non-zero if there was any error. `, DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, _ []string) { if globalOptions.JSON { type jsonVersion struct { Version string `json:"version"` diff --git a/cmd/restic/delete.go b/cmd/restic/delete.go index 1b7937bd3..c3a7e039d 100644 --- a/cmd/restic/delete.go +++ b/cmd/restic/delete.go @@ -3,9 +3,6 @@ package main import ( "context" - "golang.org/x/sync/errgroup" - - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/restic" ) @@ -24,46 +21,21 @@ func DeleteFilesChecked(ctx context.Context, gopts GlobalOptions, repo restic.Re // deleteFiles deletes the given fileList of fileType in parallel // if ignoreError=true, it will print a warning if there was an error, else it will abort. func deleteFiles(ctx context.Context, gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error { - totalCount := len(fileList) - fileChan := make(chan restic.ID) - wg, ctx := errgroup.WithContext(ctx) - wg.Go(func() error { - defer close(fileChan) - for id := range fileList { - select { - case fileChan <- id: - case <-ctx.Done(): - return ctx.Err() + bar := newProgressMax(!gopts.JSON && !gopts.Quiet, 0, "files deleted") + defer bar.Done() + + return restic.ParallelRemove(ctx, repo, fileList, fileType, func(id restic.ID, err error) error { + if err != nil { + if !gopts.JSON { + Warnf("unable to remove %v/%v from the repository\n", fileType, id) + } + if !ignoreError { + return err } } + if !gopts.JSON && gopts.verbosity > 2 { + Verbosef("removed %v/%v\n", fileType, id) + } return nil - }) - - bar := newProgressMax(!gopts.JSON && !gopts.Quiet, uint64(totalCount), "files deleted") - defer bar.Done() - // deleting files is IO-bound - workerCount := repo.Connections() - for i := 0; i < int(workerCount); i++ { - wg.Go(func() error { - for id := range fileChan { - h := backend.Handle{Type: fileType, Name: id.String()} - err := repo.Backend().Remove(ctx, h) - if err != nil { - if !gopts.JSON { - Warnf("unable to remove %v from the repository\n", h) - } - if !ignoreError { - return err - } - } - if !gopts.JSON && gopts.verbosity > 2 { - Verbosef("removed %v\n", h) - } - bar.Add(1) - } - return nil - }) - } - err := wg.Wait() - return err + }, bar) } diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go index 095944610..d9bb63aeb 100644 --- a/cmd/restic/exclude.go +++ b/cmd/restic/exclude.go @@ -426,7 +426,7 @@ func readExcludePatternsFromFiles(excludeFiles []string) ([]string, error) { return scanner.Err() }() if err != nil { - return nil, err + return nil, fmt.Errorf("failed to read excludes from file %q: %w", filename, err) } } return excludes, nil diff --git a/cmd/restic/global.go b/cmd/restic/global.go index d94c5fd1b..ff060ff1a 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -44,7 +44,7 @@ import ( "golang.org/x/term" ) -var version = "0.16.2-dev (compiled manually)" +var version = "0.16.4-dev (compiled manually)" // TimeFormat is the format used for all timestamps printed by restic. const TimeFormat = "2006-01-02 15:04:05" @@ -68,6 +68,7 @@ type GlobalOptions struct { CleanupCache bool Compression repository.CompressionMode PackSize uint + NoExtraVerify bool backend.TransportOptions limiter.Limits @@ -141,6 +142,7 @@ func init() { f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)") f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)") + f.BoolVar(&globalOptions.NoExtraVerify, "no-extra-verify", false, "skip additional verification of data before upload (see documentation)") f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)") f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)") f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)") @@ -455,8 +457,9 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi } s, err := repository.New(be, repository.Options{ - Compression: opts.Compression, - PackSize: opts.PackSize * 1024 * 1024, + Compression: opts.Compression, + PackSize: opts.PackSize * 1024 * 1024, + NoExtraVerify: opts.NoExtraVerify, }) if err != nil { return nil, errors.Fatal(err.Error()) diff --git a/cmd/restic/main.go b/cmd/restic/main.go index 4595e8161..b31ce1bb4 100644 --- a/cmd/restic/main.go +++ b/cmd/restic/main.go @@ -37,7 +37,7 @@ The full documentation can be found at https://restic.readthedocs.io/ . SilenceUsage: true, DisableAutoGenTag: true, - PersistentPreRunE: func(c *cobra.Command, args []string) error { + PersistentPreRunE: func(c *cobra.Command, _ []string) error { // set verbosity, default is one globalOptions.verbosity = 1 if globalOptions.Quiet && globalOptions.Verbose > 0 { diff --git a/cmd/restic/progress.go b/cmd/restic/progress.go index 8b33f94c9..48aa209a6 100644 --- a/cmd/restic/progress.go +++ b/cmd/restic/progress.go @@ -30,7 +30,7 @@ func calculateProgressInterval(show bool, json bool) time.Duration { } // newTerminalProgressMax returns a progress.Counter that prints to stdout or terminal if provided. -func newGenericProgressMax(show bool, max uint64, description string, print func(status string)) *progress.Counter { +func newGenericProgressMax(show bool, max uint64, description string, print func(status string, final bool)) *progress.Counter { if !show { return nil } @@ -46,16 +46,18 @@ func newGenericProgressMax(show bool, max uint64, description string, print func ui.FormatDuration(d), ui.FormatPercent(v, max), v, max, description) } - print(status) - if final { - fmt.Print("\n") - } + print(status, final) }) } func newTerminalProgressMax(show bool, max uint64, description string, term *termstatus.Terminal) *progress.Counter { - return newGenericProgressMax(show, max, description, func(status string) { - term.SetStatus([]string{status}) + return newGenericProgressMax(show, max, description, func(status string, final bool) { + if final { + term.SetStatus([]string{}) + term.Print(status) + } else { + term.SetStatus([]string{status}) + } }) } @@ -64,7 +66,7 @@ func newProgressMax(show bool, max uint64, description string) *progress.Counter return newGenericProgressMax(show, max, description, printProgress) } -func printProgress(status string) { +func printProgress(status string, final bool) { canUpdateStatus := stdoutCanUpdateStatus() @@ -95,6 +97,9 @@ func printProgress(status string) { } _, _ = os.Stdout.Write([]byte(clear + status + carriageControl)) + if final { + _, _ = os.Stdout.Write([]byte("\n")) + } } func newIndexProgress(quiet bool, json bool) *progress.Counter { @@ -104,3 +109,21 @@ func newIndexProgress(quiet bool, json bool) *progress.Counter { func newIndexTerminalProgress(quiet bool, json bool, term *termstatus.Terminal) *progress.Counter { return newTerminalProgressMax(!quiet && !json && stdoutIsTerminal(), 0, "index files loaded", term) } + +type terminalProgressPrinter struct { + term *termstatus.Terminal + ui.Message + show bool +} + +func (t *terminalProgressPrinter) NewCounter(description string) *progress.Counter { + return newTerminalProgressMax(t.show, 0, description, t.term) +} + +func newTerminalProgressPrinter(verbosity uint, term *termstatus.Terminal) progress.Printer { + return &terminalProgressPrinter{ + term: term, + Message: *ui.NewMessage(term, verbosity), + show: verbosity > 0, + } +} diff --git a/cmd/restic/termstatus.go b/cmd/restic/termstatus.go new file mode 100644 index 000000000..cf3cd82ee --- /dev/null +++ b/cmd/restic/termstatus.go @@ -0,0 +1,43 @@ +package main + +import ( + "context" + "sync" + + "github.com/restic/restic/internal/ui" + "github.com/restic/restic/internal/ui/termstatus" +) + +// setupTermstatus creates a new termstatus and reroutes globalOptions.{stdout,stderr} to it +// The returned function must be called to shut down the termstatus, +// +// Expected usage: +// ``` +// term, cancel := setupTermstatus() +// defer cancel() +// // do stuff +// ``` +func setupTermstatus() (*termstatus.Terminal, func()) { + var wg sync.WaitGroup + // only shutdown once cancel is called to ensure that no output is lost + cancelCtx, cancel := context.WithCancel(context.Background()) + + term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet) + wg.Add(1) + go func() { + defer wg.Done() + term.Run(cancelCtx) + }() + + // use the termstatus for stdout/stderr + prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr + stdioWrapper := ui.NewStdioWrapper(term) + globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr() + + return term, func() { + // shutdown termstatus + globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr + cancel() + wg.Wait() + } +} diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index 6c89e356c..b7bb2a44b 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -35,15 +35,15 @@ environment variable ``RESTIC_REPOSITORY_FILE``. For automating the supply of the repository password to restic, several options exist: - * Setting the environment variable ``RESTIC_PASSWORD`` +* Setting the environment variable ``RESTIC_PASSWORD`` - * Specifying the path to a file with the password via the option - ``--password-file`` or the environment variable ``RESTIC_PASSWORD_FILE`` +* Specifying the path to a file with the password via the option + ``--password-file`` or the environment variable ``RESTIC_PASSWORD_FILE`` + +* Configuring a program to be called when the password is needed via the + option ``--password-command`` or the environment variable + ``RESTIC_PASSWORD_COMMAND`` - * Configuring a program to be called when the password is needed via the - option ``--password-command`` or the environment variable - ``RESTIC_PASSWORD_COMMAND`` - The ``init`` command has an option called ``--repository-version`` which can be used to explicitly set the version of the new repository. By default, the current stable version is used (see table below). The alias ``latest`` will @@ -487,7 +487,8 @@ Backblaze B2 Different from the B2 backend, restic's S3 backend will only hide no longer necessary files. Thus, make sure to setup lifecycle rules to eventually - delete hidden files. + delete hidden files. The lifecycle setting "Keep only the last version of the file" + will keep only the most current version of a file. Read the [Backblaze documentation](https://www.backblaze.com/docs/cloud-storage-lifecycle-rules). Restic can backup data to any Backblaze B2 bucket. You need to first setup the following environment variables with the credentials you can find in the @@ -548,6 +549,14 @@ For authentication export one of the following variables: # For SAS $ export AZURE_ACCOUNT_SAS= +For authentication using ``az login`` set the resource group name and ensure the user has +the minimum permissions of the role assignment ``Storage Blob Data Contributor`` on Azure RBAC. + +.. code-block:: console + + $ export AZURE_RESOURCE_GROUP= + $ az login + Alternatively, if run on Azure, restic will automatically uses service accounts configured via the standard environment variables or Workload / Managed Identities. @@ -736,9 +745,9 @@ For debugging rclone, you can set the environment variable ``RCLONE_VERBOSE=2``. The rclone backend has three additional options: - * ``-o rclone.program`` specifies the path to rclone, the default value is just ``rclone`` - * ``-o rclone.args`` allows setting the arguments passed to rclone, by default this is ``serve restic --stdio --b2-hard-delete`` - * ``-o rclone.timeout`` specifies timeout for waiting on repository opening, the default value is ``1m`` +* ``-o rclone.program`` specifies the path to rclone, the default value is just ``rclone`` +* ``-o rclone.args`` allows setting the arguments passed to rclone, by default this is ``serve restic --stdio --b2-hard-delete`` +* ``-o rclone.timeout`` specifies timeout for waiting on repository opening, the default value is ``1m`` The reason for the ``--b2-hard-delete`` parameters can be found in the corresponding GitHub `issue #1657`_. diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 28a8e4b97..7f87bc9ee 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -170,10 +170,10 @@ On **Unix** (including Linux and Mac), given that a file lives at the same location as a file in a previous backup, the following file metadata attributes have to match for its contents to be presumed unchanged: - * Modification timestamp (mtime). - * Metadata change timestamp (ctime). - * File size. - * Inode number (internal number used to reference a file in a filesystem). +* Modification timestamp (mtime). +* Metadata change timestamp (ctime). +* File size. +* Inode number (internal number used to reference a file in a filesystem). The reason for requiring both mtime and ctime to match is that Unix programs can freely change mtime (and some do). In such cases, a ctime change may be @@ -182,9 +182,9 @@ the only hint that a file did change. The following ``restic backup`` command line flags modify the change detection rules: - * ``--force``: turn off change detection and rescan all files. - * ``--ignore-ctime``: require mtime to match, but allow ctime to differ. - * ``--ignore-inode``: require mtime to match, but allow inode number +* ``--force``: turn off change detection and rescan all files. +* ``--ignore-ctime``: require mtime to match, but allow ctime to differ. +* ``--ignore-inode``: require mtime to match, but allow inode number and ctime to differ. The option ``--ignore-inode`` exists to support FUSE-based filesystems and @@ -250,9 +250,9 @@ It can be used like this: This instructs restic to exclude files matching the following criteria: - * All files matching ``*.c`` (parameter ``--exclude``) - * All files matching ``*.go`` (second line in ``excludes.txt``) - * All files and sub-directories named ``bar`` which reside somewhere below a directory called ``foo`` (fourth line in ``excludes.txt``) +* All files matching ``*.c`` (parameter ``--exclude``) +* All files matching ``*.go`` (second line in ``excludes.txt``) +* All files and sub-directories named ``bar`` which reside somewhere below a directory called ``foo`` (fourth line in ``excludes.txt``) Patterns use the syntax of the Go function `filepath.Match `__ @@ -270,8 +270,8 @@ environment variable (depending on your operating system). Patterns need to match on complete path components. For example, the pattern ``foo``: - * matches ``/dir1/foo/dir2/file`` and ``/dir/foo`` - * does not match ``/dir/foobar`` or ``barfoo`` +* matches ``/dir1/foo/dir2/file`` and ``/dir/foo`` +* does not match ``/dir/foobar`` or ``barfoo`` A trailing ``/`` is ignored, a leading ``/`` anchors the pattern at the root directory. This means, ``/bin`` matches ``/bin/bash`` but does not match ``/usr/bin/restic``. @@ -281,9 +281,9 @@ e.g. ``b*ash`` matches ``/bin/bash`` but does not match ``/bin/ash``. For this, the special wildcard ``**`` can be used to match arbitrary sub-directories: The pattern ``foo/**/bar`` matches: - * ``/dir1/foo/dir2/bar/file`` - * ``/foo/bar/file`` - * ``/tmp/foo/bar`` +* ``/dir1/foo/dir2/bar/file`` +* ``/foo/bar/file`` +* ``/tmp/foo/bar`` Spaces in patterns listed in an exclude file can be specified verbatim. That is, in order to exclude a file named ``foo bar star.txt``, put that just as it reads @@ -298,9 +298,9 @@ some escaping in order to pass the name/pattern as a single argument to restic. On most Unixy shells, you can either quote or use backslashes. For example: - * ``--exclude='foo bar star/foo.txt'`` - * ``--exclude="foo bar star/foo.txt"`` - * ``--exclude=foo\ bar\ star/foo.txt`` +* ``--exclude='foo bar star/foo.txt'`` +* ``--exclude="foo bar star/foo.txt"`` +* ``--exclude=foo\ bar\ star/foo.txt`` If a pattern starts with exclamation mark and matches a file that was previously matched by a regular pattern, the match is cancelled. @@ -381,8 +381,8 @@ contains one *pattern* per line. The file must be encoded as UTF-8, or UTF-16 with a byte-order mark. Leading and trailing whitespace is removed from the patterns. Empty lines and lines starting with a ``#`` are ignored and each pattern is expanded when read, such that special characters in it are expanded -using the Go function `filepath.Glob `__ -- please see its documentation for the syntax you can use in the patterns. +according to the syntax described in the documentation of the Go function +`filepath.Match `__. The argument passed to ``--files-from-verbatim`` must be the name of a text file that contains one *path* per line, e.g. as generated by GNU ``find`` with the @@ -482,13 +482,11 @@ want to save the access time for files and directories, you can pass the ``--with-atime`` option to the ``backup`` command. Note that ``restic`` does not back up some metadata associated with files. Of -particular note are:: - - - file creation date on Unix platforms - - inode flags on Unix platforms - - file ownership and ACLs on Windows - - the "hidden" flag on Windows +particular note are: +* File creation date on Unix platforms +* Inode flags on Unix platforms +* File ownership and ACLs on Windows Reading data from a command *************************** @@ -514,7 +512,6 @@ Restic uses the command exit code to determine whether the command succeeded. A non-zero exit code from the command causes restic to cancel the backup. This causes restic to fail with exit code 1. No snapshot will be created in this case. - Reading data from stdin *********************** @@ -555,7 +552,6 @@ the pipe and act accordingly (e.g., remove the last backup). Refer to the `Use the Unofficial Bash Strict Mode `__ for more details on this. - Tags for backup *************** @@ -688,15 +684,14 @@ The external programs that restic may execute include ``rclone`` (for rclone backends) and ``ssh`` (for the SFTP backend). These may respond to further environment variables and configuration files; see their respective manuals. - Exit status codes ***************** Restic returns one of the following exit status codes after the backup command is run: - * 0 when the backup was successful (snapshot with all source files created) - * 1 when there was a fatal error (no snapshot created) - * 3 when some source files could not be read (incomplete snapshot with remaining files created) +* 0 when the backup was successful (snapshot with all source files created) +* 1 when there was a fatal error (no snapshot created) +* 3 when some source files could not be read (incomplete snapshot with remaining files created) Fatal errors occur for example when restic is unable to write to the backup destination, when there are network connectivity issues preventing successful communication, or when an invalid diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index d74c9c240..48e5985dc 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -82,6 +82,76 @@ Furthermore you can group the output by the same filters (host, paths, tags): 1 snapshots +Listing files in a snapshot +=========================== + +To get a list of the files in a specific snapshot you can use the ``ls`` command: + +.. code-block:: console + + $ restic ls 073a90db + + snapshot 073a90db of [/home/user/work.txt] filtered by [] at 2024-01-21 16:51:18.474558607 +0100 CET): + /home + /home/user + /home/user/work.txt + +The special snapshot ID ``latest`` can be used to list files and directories of the latest snapshot in the repository. +The ``--host`` flag can be used in conjunction to select the latest snapshot originating from a certain host only. + +.. code-block:: console + + $ restic ls --host kasimir latest + + snapshot 073a90db of [/home/user/work.txt] filtered by [] at 2024-01-21 16:51:18.474558607 +0100 CET): + /home + /home/user + /home/user/work.txt + +By default, ``ls`` prints all files in a snapshot. + +File listings can optionally be filtered by directories. Any positional arguments after the snapshot ID are interpreted +as absolute directory paths, and only files inside those directories will be listed. Files in subdirectories are not +listed when filtering by directories. If the ``--recursive`` flag is used, then subdirectories are also included. +Any directory paths specified must be absolute (starting with a path separator); paths use the forward slash '/' +as separator. + +.. code-block:: console + + $ restic ls latest /home + + snapshot 073a90db of [/home/user/work.txt] filtered by [/home] at 2024-01-21 16:51:18.474558607 +0100 CET): + /home + /home/user + +.. code-block:: console + + $ restic ls --recursive latest /home + + snapshot 073a90db of [/home/user/work.txt] filtered by [/home] at 2024-01-21 16:51:18.474558607 +0100 CET): + /home + /home/user + /home/user/work.txt + +To show more details about the files in a snapshot, you can use the ``--long`` option. The colums include +file permissions, UID, GID, file size, modification time and file path. For scripting usage, the +``ls`` command supports the ``--json`` flag; the JSON output format is described at :ref:`ls json`. + +.. code-block:: console + + $ restic ls --long latest + + snapshot 073a90db of [/home/user/work.txt] filtered by [] at 2024-01-21 16:51:18.474558607 +0100 CET): + drwxr-xr-x 0 0 0 2024-01-21 16:50:52 /home + drwxr-xr-x 0 0 0 2024-01-21 16:51:03 /home/user + -rw-r--r-- 0 0 18 2024-01-21 16:51:03 /home/user/work.txt + +NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories. The ``ls`` command supports +outputting information about a snapshot in the NCDU format using the ``--ncdu`` option. + +You can use it as follows: ``restic ls latest --ncdu | ncdu -f -`` + + Copying snapshots between repositories ====================================== @@ -242,6 +312,7 @@ Currently, rewriting the hostname and the time of the backup is supported. This is possible using the ``rewrite`` command with the option ``--new-host`` followed by the desired new hostname or the option ``--new-time`` followed by the desired new timestamp. .. code-block:: console + $ restic rewrite --new-host newhost --new-time "1999-01-01 11:11:11" repository b7dbade3 opened (version 2, compression level auto) diff --git a/doc/047_tuning_backup_parameters.rst b/doc/047_tuning_backup_parameters.rst index 6ea39dc75..d8fb2c9b6 100644 --- a/doc/047_tuning_backup_parameters.rst +++ b/doc/047_tuning_backup_parameters.rst @@ -60,6 +60,20 @@ only applied for the single run of restic. The option can also be set via the en variable ``RESTIC_COMPRESSION``. +Data Verification +================= + +To prevent the upload of corrupted data to the repository, which can happen due +to hardware issues or software bugs, restic verifies that generated files can +be decoded and contain the correct data beforehand. This increases the CPU usage +during backups. If necessary, you can disable this verification using the +``--no-extra-verify`` option of the ``backup`` command. However, in this case +you should verify the repository integrity more actively using +``restic check --read-data`` (or the similar ``--read-data-subset`` option). +Otherwise, data corruption due to hardware issues or software bugs might go +unnoticed. + + File Read Concurrency ===================== diff --git a/doc/050_restore.rst b/doc/050_restore.rst index 56f6458ed..916b11c86 100644 --- a/doc/050_restore.rst +++ b/doc/050_restore.rst @@ -174,3 +174,9 @@ To include the folder content at the root of the archive, you can use the `` restore.tar + +It is also possible to ``dump`` the contents of a selected snapshot and folder +structure to a file using the ``--target`` flag. + +.. code-block:: console + $ restic -r /srv/restic-repo dump latest / --target /home/linux.user/output.tar -a tar \ No newline at end of file diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index f46572209..fda4b2d53 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -75,9 +75,6 @@ Several commands, in particular long running ones or those that generate a large use a format also known as JSON lines. It consists of a stream of new-line separated JSON messages. You can determine the nature of the message using the ``message_type`` field. -As an exception, the ``ls`` command uses the field ``struct_type`` instead. - - backup ------ @@ -409,6 +406,8 @@ The ``key list`` command returns an array of objects with the following structur +--------------+------------------------------------+ +.. _ls json: + ls -- @@ -418,63 +417,67 @@ As an exception, the ``struct_type`` field is used to determine the message type snapshot ^^^^^^^^ -+----------------+--------------------------------------------------+ -| ``struct_type``| Always "snapshot" | -+----------------+--------------------------------------------------+ -| ``time`` | Timestamp of when the backup was started | -+----------------+--------------------------------------------------+ -| ``parent`` | ID of the parent snapshot | -+----------------+--------------------------------------------------+ -| ``tree`` | ID of the root tree blob | -+----------------+--------------------------------------------------+ -| ``paths`` | List of paths included in the backup | -+----------------+--------------------------------------------------+ -| ``hostname`` | Hostname of the backed up machine | -+----------------+--------------------------------------------------+ -| ``username`` | Username the backup command was run as | -+----------------+--------------------------------------------------+ -| ``uid`` | ID of owner | -+----------------+--------------------------------------------------+ -| ``gid`` | ID of group | -+----------------+--------------------------------------------------+ -| ``excludes`` | List of paths and globs excluded from the backup | -+----------------+--------------------------------------------------+ -| ``tags`` | List of tags for the snapshot in question | -+----------------+--------------------------------------------------+ -| ``id`` | Snapshot ID | -+----------------+--------------------------------------------------+ -| ``short_id`` | Snapshot ID, short form | -+----------------+--------------------------------------------------+ ++------------------+--------------------------------------------------+ +| ``message_type`` | Always "snapshot" | ++------------------+--------------------------------------------------+ +| ``struct_type`` | Always "snapshot" (deprecated) | ++------------------+--------------------------------------------------+ +| ``time`` | Timestamp of when the backup was started | ++------------------+--------------------------------------------------+ +| ``parent`` | ID of the parent snapshot | ++------------------+--------------------------------------------------+ +| ``tree`` | ID of the root tree blob | ++------------------+--------------------------------------------------+ +| ``paths`` | List of paths included in the backup | ++------------------+--------------------------------------------------+ +| ``hostname`` | Hostname of the backed up machine | ++------------------+--------------------------------------------------+ +| ``username`` | Username the backup command was run as | ++------------------+--------------------------------------------------+ +| ``uid`` | ID of owner | ++------------------+--------------------------------------------------+ +| ``gid`` | ID of group | ++------------------+--------------------------------------------------+ +| ``excludes`` | List of paths and globs excluded from the backup | ++------------------+--------------------------------------------------+ +| ``tags`` | List of tags for the snapshot in question | ++------------------+--------------------------------------------------+ +| ``id`` | Snapshot ID | ++------------------+--------------------------------------------------+ +| ``short_id`` | Snapshot ID, short form | ++------------------+--------------------------------------------------+ node ^^^^ -+-----------------+--------------------------+ -| ``struct_type`` | Always "node" | -+-----------------+--------------------------+ -| ``name`` | Node name | -+-----------------+--------------------------+ -| ``type`` | Node type | -+-----------------+--------------------------+ -| ``path`` | Node path | -+-----------------+--------------------------+ -| ``uid`` | UID of node | -+-----------------+--------------------------+ -| ``gid`` | GID of node | -+-----------------+--------------------------+ -| ``size`` | Size in bytes | -+-----------------+--------------------------+ -| ``mode`` | Node mode | -+-----------------+--------------------------+ -| ``atime`` | Node access time | -+-----------------+--------------------------+ -| ``mtime`` | Node modification time | -+-----------------+--------------------------+ -| ``ctime`` | Node creation time | -+-----------------+--------------------------+ -| ``inode`` | Inode number of node | -+-----------------+--------------------------+ ++------------------+----------------------------+ +| ``message_type`` | Always "node" | ++------------------+----------------------------+ +| ``struct_type`` | Always "node" (deprecated) | ++------------------+----------------------------+ +| ``name`` | Node name | ++------------------+----------------------------+ +| ``type`` | Node type | ++------------------+----------------------------+ +| ``path`` | Node path | ++------------------+----------------------------+ +| ``uid`` | UID of node | ++------------------+----------------------------+ +| ``gid`` | GID of node | ++------------------+----------------------------+ +| ``size`` | Size in bytes | ++------------------+----------------------------+ +| ``mode`` | Node mode | ++------------------+----------------------------+ +| ``atime`` | Node access time | ++------------------+----------------------------+ +| ``mtime`` | Node modification time | ++------------------+----------------------------+ +| ``ctime`` | Node creation time | ++------------------+----------------------------+ +| ``inode`` | Inode number of node | ++------------------+----------------------------+ restore diff --git a/doc/077_troubleshooting.rst b/doc/077_troubleshooting.rst index 6a9a6ee15..f80df29b8 100644 --- a/doc/077_troubleshooting.rst +++ b/doc/077_troubleshooting.rst @@ -76,6 +76,10 @@ Similarly, if a repository is repeatedly damaged, please open an `issue on Githu somewhere. Please include the check output and additional information that might help locate the problem. +If ``check`` detects damaged pack files, it will show instructions on how to repair +them using the ``repair pack`` command. Use that command instead of the "Repair the +index" section in this guide. + 2. Backup the repository ************************ @@ -104,6 +108,11 @@ whether your issue is already known and solved. Please take a look at the 3. Repair the index ******************* +.. note:: + + If the `check` command tells you to run `restic repair pack`, then use that + command instead. It will repair the damaged pack files and also update the index. + Restic relies on its index to contain correct information about what data is stored in the repository. Thus, the first step to repair a repository is to repair the index: diff --git a/doc/REST_backend.rst b/doc/REST_backend.rst index f9d72cf06..9e85187f9 100644 --- a/doc/REST_backend.rst +++ b/doc/REST_backend.rst @@ -7,18 +7,18 @@ API. The following values are valid for ``{type}``: - * ``data`` - * ``keys`` - * ``locks`` - * ``snapshots`` - * ``index`` - * ``config`` +* ``data`` +* ``keys`` +* ``locks`` +* ``snapshots`` +* ``index`` +* ``config`` The API version is selected via the ``Accept`` HTTP header in the request. The following values are defined: - * ``application/vnd.x.restic.rest.v1`` or empty: Select API version 1 - * ``application/vnd.x.restic.rest.v2``: Select API version 2 +* ``application/vnd.x.restic.rest.v1`` or empty: Select API version 1 +* ``application/vnd.x.restic.rest.v2``: Select API version 2 The server will respond with the value of the highest version it supports in the ``Content-Type`` HTTP response header for the HTTP requests which should diff --git a/doc/bash-completion.sh b/doc/bash-completion.sh index e691af363..cae37a6ca 100644 --- a/doc/bash-completion.sh +++ b/doc/bash-completion.sh @@ -488,6 +488,7 @@ _restic_backup() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -560,6 +561,7 @@ _restic_cache() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -624,6 +626,7 @@ _restic_cat() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -696,6 +699,7 @@ _restic_check() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -794,6 +798,7 @@ _restic_copy() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -860,6 +865,7 @@ _restic_diff() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -944,6 +950,7 @@ _restic_dump() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1058,6 +1065,7 @@ _restic_find() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1228,6 +1236,7 @@ _restic_forget() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1312,6 +1321,7 @@ _restic_generate() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1372,6 +1382,7 @@ _restic_help() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1463,6 +1474,7 @@ _restic_init() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1539,6 +1551,7 @@ _restic_key() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1603,6 +1616,7 @@ _restic_list() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1689,6 +1703,7 @@ _restic_ls() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1757,6 +1772,7 @@ _restic_migrate() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1849,6 +1865,7 @@ _restic_mount() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1935,6 +1952,7 @@ _restic_prune() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1999,6 +2017,7 @@ _restic_recover() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2059,6 +2078,7 @@ _restic_repair_help() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2126,6 +2146,7 @@ _restic_repair_index() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2190,6 +2211,7 @@ _restic_repair_packs() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2274,6 +2296,7 @@ _restic_repair_snapshots() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2342,6 +2365,7 @@ _restic_repair() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2450,6 +2474,7 @@ _restic_restore() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2552,6 +2577,7 @@ _restic_rewrite() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2620,6 +2646,7 @@ _restic_self-update() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2712,6 +2739,7 @@ _restic_snapshots() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2794,6 +2822,7 @@ _restic_stats() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2884,6 +2913,7 @@ _restic_tag() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2950,6 +2980,7 @@ _restic_unlock() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -3014,6 +3045,7 @@ _restic_version() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -3106,6 +3138,7 @@ _restic_root_command() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") diff --git a/doc/design.rst b/doc/design.rst index 1e00a3358..b80029d10 100644 --- a/doc/design.rst +++ b/doc/design.rst @@ -824,4 +824,4 @@ Changes Repository Version 2 -------------------- - * Support compression for blobs (data/tree) and index / lock / snapshot files +* Support compression for blobs (data/tree) and index / lock / snapshot files diff --git a/doc/developer_information.rst b/doc/developer_information.rst index 9de517901..c7757e087 100644 --- a/doc/developer_information.rst +++ b/doc/developer_information.rst @@ -9,14 +9,14 @@ restic for version 0.10.0 and later. For restic versions down to 0.9.3 please refer to the documentation for the respective version. The binary produced depends on the following things: - * The source code for the release - * The exact version of the official `Go compiler `__ used to produce the binaries (running ``restic version`` will print this) - * The architecture and operating system the Go compiler runs on (Linux, ``amd64``) - * The build tags (for official binaries, it's the tag ``selfupdate``) - * The path where the source code is extracted to (``/restic``) - * The path to the Go compiler (``/usr/local/go``) - * The path to the Go workspace (``GOPATH=/home/build/go``) - * Other environment variables (mostly ``$GOOS``, ``$GOARCH``, ``$CGO_ENABLED``) +* The source code for the release +* The exact version of the official `Go compiler `__ used to produce the binaries (running ``restic version`` will print this) +* The architecture and operating system the Go compiler runs on (Linux, ``amd64``) +* The build tags (for official binaries, it's the tag ``selfupdate``) +* The path where the source code is extracted to (``/restic``) +* The path to the Go compiler (``/usr/local/go``) +* The path to the Go workspace (``GOPATH=/home/build/go``) +* Other environment variables (mostly ``$GOOS``, ``$GOARCH``, ``$CGO_ENABLED``) In addition, The compressed ZIP files for Windows depends on the modification timestamp and filename of the binary contained in it. In order to reproduce the @@ -69,9 +69,9 @@ container can be found in the `GitHub repository `__ The container serves the following goals: - * Have a very controlled environment which is independent from the local system - * Make it easy to have the correct version of the Go compiler at the right path - * Make it easy to pass in the source code to build at a well-defined path +* Have a very controlled environment which is independent from the local system +* Make it easy to have the correct version of the Go compiler at the right path +* Make it easy to pass in the source code to build at a well-defined path The following steps are necessary to build the binaries: @@ -113,6 +113,26 @@ The following steps are necessary to build the binaries: restic/builder \ go run helpers/build-release-binaries/main.go --version 0.14.0 --verbose +Verifying the Official Binaries +******************************* + +To verify the official binaries, you can either build them yourself using the above +instructions or use the ``helpers/verify-release-binaries.sh`` script from the restic +repository. Run it as ``helpers/verify-release-binaries.sh restic_version go_version``. +The specified go compiler version must match the one used to build the official +binaries. For example, for restic 0.16.2 the command would be +``helpers/verify-release-binaries.sh 0.16.2 1.21.3``. + +The script requires bash, curl, docker, git, gpg, shasum and tar. + +The script first downloads all release binaries, checks the SHASUM256 file and its +signature. Afterwards it checks that the tarball matches the restic git repository +contents, before first reproducing the builder docker container and finally the +restic binaries. As final step, the restic binary in both the docker hub images +and the GitHub container registry is verified. If any step fails, then the script +will issue a warning. + + Prepare a New Release ********************* diff --git a/doc/man/restic-backup.1 b/doc/man/restic-backup.1 index c3bccdfa5..730685271 100644 --- a/doc/man/restic-backup.1 +++ b/doc/man/restic-backup.1 @@ -171,6 +171,10 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-cache.1 b/doc/man/restic-cache.1 index 3ae27ea57..c170c1624 100644 --- a/doc/man/restic-cache.1 +++ b/doc/man/restic-cache.1 @@ -80,6 +80,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-cat.1 b/doc/man/restic-cat.1 index c1df138aa..b42a58e14 100644 --- a/doc/man/restic-cat.1 +++ b/doc/man/restic-cat.1 @@ -68,6 +68,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-check.1 b/doc/man/restic-check.1 index 17eb972bc..9c1dc77e5 100644 --- a/doc/man/restic-check.1 +++ b/doc/man/restic-check.1 @@ -85,6 +85,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-copy.1 b/doc/man/restic-copy.1 index be8f21e25..bd9795f44 100644 --- a/doc/man/restic-copy.1 +++ b/doc/man/restic-copy.1 @@ -109,6 +109,10 @@ new destination repository using the "init" command. \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-diff.1 b/doc/man/restic-diff.1 index a01a2562b..28f3a4838 100644 --- a/doc/man/restic-diff.1 +++ b/doc/man/restic-diff.1 @@ -93,6 +93,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-dump.1 b/doc/man/restic-dump.1 index 6fa1f8200..7fa3f777d 100644 --- a/doc/man/restic-dump.1 +++ b/doc/man/restic-dump.1 @@ -96,6 +96,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-find.1 b/doc/man/restic-find.1 index 72bc3a0b6..c3297c43f 100644 --- a/doc/man/restic-find.1 +++ b/doc/man/restic-find.1 @@ -117,6 +117,10 @@ It can also be used to search for restic blobs or trees for troubleshooting. \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-forget.1 b/doc/man/restic-forget.1 index 757022a21..d0c4cfc74 100644 --- a/doc/man/restic-forget.1 +++ b/doc/man/restic-forget.1 @@ -179,6 +179,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-generate.1 b/doc/man/restic-generate.1 index aef3a5e55..84f659ef2 100644 --- a/doc/man/restic-generate.1 +++ b/doc/man/restic-generate.1 @@ -89,6 +89,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-init.1 b/doc/man/restic-init.1 index 27d7f5874..5f19c8f8c 100644 --- a/doc/man/restic-init.1 +++ b/doc/man/restic-init.1 @@ -96,6 +96,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-key.1 b/doc/man/restic-key.1 index 855ef5443..8d1813188 100644 --- a/doc/man/restic-key.1 +++ b/doc/man/restic-key.1 @@ -80,6 +80,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-list.1 b/doc/man/restic-list.1 index 95eeac5f7..e399038a2 100644 --- a/doc/man/restic-list.1 +++ b/doc/man/restic-list.1 @@ -68,6 +68,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-ls.1 b/doc/man/restic-ls.1 index 0cd0f5a88..10b0657a3 100644 --- a/doc/man/restic-ls.1 +++ b/doc/man/restic-ls.1 @@ -107,6 +107,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-migrate.1 b/doc/man/restic-migrate.1 index eca0ef8e1..7e48f726c 100644 --- a/doc/man/restic-migrate.1 +++ b/doc/man/restic-migrate.1 @@ -74,6 +74,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-mount.1 b/doc/man/restic-mount.1 index 33c016ffa..aab607fcf 100644 --- a/doc/man/restic-mount.1 +++ b/doc/man/restic-mount.1 @@ -144,6 +144,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-prune.1 b/doc/man/restic-prune.1 index e4a32cac3..c54d5d7ff 100644 --- a/doc/man/restic-prune.1 +++ b/doc/man/restic-prune.1 @@ -97,6 +97,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-recover.1 b/doc/man/restic-recover.1 index 26d2fc7bd..010fbafd7 100644 --- a/doc/man/restic-recover.1 +++ b/doc/man/restic-recover.1 @@ -70,6 +70,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-repair-index.1 b/doc/man/restic-repair-index.1 index 35e2845b8..f06be64c0 100644 --- a/doc/man/restic-repair-index.1 +++ b/doc/man/restic-repair-index.1 @@ -73,6 +73,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-repair-packs.1 b/doc/man/restic-repair-packs.1 index b21211925..f3671fe18 100644 --- a/doc/man/restic-repair-packs.1 +++ b/doc/man/restic-repair-packs.1 @@ -72,6 +72,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-repair-snapshots.1 b/doc/man/restic-repair-snapshots.1 index f59067f05..9369f25f2 100644 --- a/doc/man/restic-repair-snapshots.1 +++ b/doc/man/restic-repair-snapshots.1 @@ -107,6 +107,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-repair.1 b/doc/man/restic-repair.1 index dbe783df4..77aecc173 100644 --- a/doc/man/restic-repair.1 +++ b/doc/man/restic-repair.1 @@ -63,6 +63,10 @@ Repair the repository \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-restore.1 b/doc/man/restic-restore.1 index d8c1b72e1..4635b1e43 100644 --- a/doc/man/restic-restore.1 +++ b/doc/man/restic-restore.1 @@ -117,6 +117,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-rewrite.1 b/doc/man/restic-rewrite.1 index 8a06aef40..d63c653e6 100644 --- a/doc/man/restic-rewrite.1 +++ b/doc/man/restic-rewrite.1 @@ -121,6 +121,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-self-update.1 b/doc/man/restic-self-update.1 index 28fd24a92..92ab5add3 100644 --- a/doc/man/restic-self-update.1 +++ b/doc/man/restic-self-update.1 @@ -75,6 +75,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-snapshots.1 b/doc/man/restic-snapshots.1 index cb34d6c8e..6203bbf2b 100644 --- a/doc/man/restic-snapshots.1 +++ b/doc/man/restic-snapshots.1 @@ -92,6 +92,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-stats.1 b/doc/man/restic-stats.1 index cf0374351..9d37163de 100644 --- a/doc/man/restic-stats.1 +++ b/doc/man/restic-stats.1 @@ -114,6 +114,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-tag.1 b/doc/man/restic-tag.1 index 162d50d29..b1468c74d 100644 --- a/doc/man/restic-tag.1 +++ b/doc/man/restic-tag.1 @@ -99,6 +99,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-unlock.1 b/doc/man/restic-unlock.1 index 0274c56e8..0b3b43f2a 100644 --- a/doc/man/restic-unlock.1 +++ b/doc/man/restic-unlock.1 @@ -72,6 +72,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-version.1 b/doc/man/restic-version.1 index 774e19453..ccc23038f 100644 --- a/doc/man/restic-version.1 +++ b/doc/man/restic-version.1 @@ -69,6 +69,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic.1 b/doc/man/restic.1 index 427ce7c65..333eab76a 100644 --- a/doc/man/restic.1 +++ b/doc/man/restic.1 @@ -65,6 +65,10 @@ The full documentation can be found at https://restic.readthedocs.io/ . \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index d1c64ba6e..bf9554e04 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -112,7 +112,7 @@ command: --iexclude pattern same as --exclude pattern but ignores the casing of filenames --iexclude-file file same as --exclude-file but ignores casing of filenames in patterns --ignore-ctime ignore ctime changes when checking for modified files - --ignore-inode ignore inode number changes when checking for modified files + --ignore-inode ignore inode number and ctime changes when checking for modified files --no-scan do not run scanner to estimate size of backup -x, --one-file-system exclude other file systems, don't cross filesystem boundaries and subvolumes --parent snapshot use this parent snapshot (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time) @@ -428,10 +428,10 @@ This allows faster operations, since meta data does not need to be loaded from a remote repository. The cache is automatically created, usually in an OS-specific cache folder: - * Linux/other: ``$XDG_CACHE_HOME/restic``, or ``~/.cache/restic`` if - ``XDG_CACHE_HOME`` is not set - * macOS: ``~/Library/Caches/restic`` - * Windows: ``%LOCALAPPDATA%/restic`` +* Linux/other: ``$XDG_CACHE_HOME/restic``, or ``~/.cache/restic`` if + ``XDG_CACHE_HOME`` is not set +* macOS: ``~/Library/Caches/restic`` +* Windows: ``%LOCALAPPDATA%/restic`` If the relevant environment variables are not set, restic exits with an error message. diff --git a/go.mod b/go.mod index 5c75bee3d..d4237b750 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,10 @@ module github.com/restic/restic require ( - cloud.google.com/go/storage v1.34.0 + cloud.google.com/go/storage v1.37.0 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 github.com/Backblaze/blazer v0.6.1 github.com/anacrolix/fuse v0.2.0 github.com/cenkalti/backoff/v4 v4.2.1 @@ -14,8 +14,8 @@ require ( github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.5.0 github.com/hashicorp/golang-lru/v2 v2.0.7 - github.com/hirochachacha/go-smb2 v1.1.0 - github.com/klauspost/compress v1.17.4 + github.com/hirochachacha/go-smb2 v1.1.0 + github.com/klauspost/compress v1.17.6 github.com/minio/minio-go/v7 v7.0.66 github.com/minio/sha256-simd v1.0.1 github.com/ncw/swift/v2 v2.0.2 @@ -27,29 +27,31 @@ require ( github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 go.uber.org/automaxprocs v1.5.3 - golang.org/x/crypto v0.17.0 - golang.org/x/net v0.19.0 - golang.org/x/oauth2 v0.15.0 - golang.org/x/sync v0.5.0 - golang.org/x/sys v0.15.0 - golang.org/x/term v0.15.0 + golang.org/x/crypto v0.18.0 + golang.org/x/net v0.20.0 + golang.org/x/oauth2 v0.16.0 + golang.org/x/sync v0.6.0 + golang.org/x/sys v0.16.0 + golang.org/x/term v0.16.0 golang.org/x/text v0.14.0 golang.org/x/time v0.5.0 - google.golang.org/api v0.149.0 + google.golang.org/api v0.157.0 ) require ( - cloud.google.com/go v0.110.9 // indirect - cloud.google.com/go/compute v1.23.1 // indirect + cloud.google.com/go v0.112.0 // indirect + cloud.google.com/go/compute v1.23.3 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.3 // indirect + cloud.google.com/go/iam v1.1.5 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/fgprof v0.9.3 // indirect - github.com/geoffgarside/ber v1.1.0 // indirect - github.com/golang-jwt/jwt/v5 v5.0.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect @@ -70,13 +72,17 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect + go.opentelemetry.io/otel v1.21.0 // indirect + go.opentelemetry.io/otel/metric v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.21.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect - google.golang.org/grpc v1.59.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240122161410-6c6643bf1457 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac // indirect + google.golang.org/grpc v1.60.1 // indirect + google.golang.org/protobuf v1.32.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 69670a93f..8059c3369 100644 --- a/go.sum +++ b/go.sum @@ -1,23 +1,23 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.110.9 h1:e7ITSqGFFk4rbz/JFIqZh3G4VEHguhAL4BQcFlWtU68= -cloud.google.com/go v0.110.9/go.mod h1:rpxevX/0Lqvlbc88b7Sc1SPNdyK1riNBTUU6JXhYNpM= -cloud.google.com/go/compute v1.23.1 h1:V97tBoDaZHb6leicZ1G6DLK2BAaZLJ/7+9BB/En3hR0= -cloud.google.com/go/compute v1.23.1/go.mod h1:CqB3xpmPKKt3OJpW2ndFIXnA9A4xAy/F3Xp1ixncW78= +cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= +cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= +cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/iam v1.1.3 h1:18tKG7DzydKWUnLjonWcJO6wjSCAtzh4GcRKlH/Hrzc= -cloud.google.com/go/iam v1.1.3/go.mod h1:3khUlaBXfPKKe7huYgEpDn6FtgRyMEqbkvBxrQyY5SE= -cloud.google.com/go/storage v1.34.0 h1:9KHBBTbaHPsNxO043SFmH3pMojjZiW+BFl9H41L7xjk= -cloud.google.com/go/storage v1.34.0/go.mod h1:Eji+S0CCQebjsiXxyIvPItC3BN3zWsdJjWfHfoLblgY= +cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= +cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= +cloud.google.com/go/storage v1.37.0 h1:WI8CsaFO8Q9KjPVtsZ5Cmi0dXV25zMoX0FklT7c3Jm4= +cloud.google.com/go/storage v1.37.0/go.mod h1:i34TiT2IhiNDmcj65PqwCjcoUX7Z5pLzS8DEmoiFq1k= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 h1:gggzg0SUMs6SQbEw+3LoSsYf9YMjkupeAnHMX8O9mmY= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0/go.mod h1:+6KLcKIVgxoBDMqMO/Nvy7bZ9a0nbU3I1DtFQK3YvB4= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 h1:AMf7YbZOZIW5b66cXNHMWWT/zkjhz5+a+k/3x40EO7E= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1/go.mod h1:uwfk06ZBcvL/g4VHNjurPfVln9NMbsk2XIZxJ+hu81k= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 h1:hVeq+yCyUi+MsoO/CU95yqCIcdzra5ovzk8Q2BBpV2M= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/Backblaze/blazer v0.6.1 h1:xC9HyC7OcxRzzmtfRiikIEvq4HZYWjU6caFwX2EXw1s= @@ -36,6 +36,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -54,10 +55,16 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= -github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= -github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= @@ -112,8 +119,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= +github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -180,6 +187,17 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -188,8 +206,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -207,18 +225,18 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= -golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -234,13 +252,13 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -262,9 +280,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -google.golang.org/api v0.149.0 h1:b2CqT6kG+zqJIVKRQ3ELJVLN1PwHZ6DJ3dW8yl82rgY= -google.golang.org/api v0.149.0/go.mod h1:Mwn1B7JTXrzXtnvmzQE2BD6bYZQ8DShKZDZbeN9I7qI= +google.golang.org/api v0.157.0 h1:ORAeqmbrrozeyw5NjnMxh7peHO0UzV4wWYSwZeCUb20= +google.golang.org/api v0.157.0/go.mod h1:+z4v4ufbZ1WEpld6yMGHyggs+PmAHiaLNj5ytP3N01g= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= @@ -272,19 +289,19 @@ google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA= -google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI= -google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k= -google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= +google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac h1:ZL/Teoy/ZGnzyrqK/Optxxp2pmVh+fmJ97slxSRyzUg= +google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:+Rvu7ElI+aLzyDQhpHMFMMltsD6m7nqpuWDd2CwJw3k= +google.golang.org/genproto/googleapis/api v0.0.0-20240122161410-6c6643bf1457 h1:KHBtwE+eQc3+NxpjmRFlQ3pJQ2FNnhhgB9xOV8kyBuU= +google.golang.org/genproto/googleapis/api v0.0.0-20240122161410-6c6643bf1457/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac h1:nUQEQmH/csSvFECKYRv6HWEyypysidKl2I6Qpsglq/0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -296,8 +313,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index f2c481b32..77ddba7c4 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -147,8 +147,8 @@ func (o Options) ApplyDefaults() Options { func New(repo restic.Repository, fs fs.FS, opts Options) *Archiver { arch := &Archiver{ Repo: repo, - SelectByName: func(item string) bool { return true }, - Select: func(item string, fi os.FileInfo) bool { return true }, + SelectByName: func(_ string) bool { return true }, + Select: func(_ string, _ os.FileInfo) bool { return true }, FS: fs, Options: opts.ApplyDefaults(), @@ -762,7 +762,7 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps arch.runWorkers(wgCtx, wg) debug.Log("starting snapshot") - fn, nodeCount, err := arch.SaveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(n *restic.Node, is ItemStats) { + fn, nodeCount, err := arch.SaveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(_ *restic.Node, is ItemStats) { arch.CompleteItem("/", nil, nil, is, time.Since(start)) }) if err != nil { diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index c6daed5bb..46ef44251 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -1880,7 +1880,7 @@ func TestArchiverContextCanceled(t *testing.T) { }) // Ensure that the archiver itself reports the canceled context and not just the backend - repo := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0) + repo := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0, repository.Options{}) back := restictest.Chdir(t, tempdir) defer back() diff --git a/internal/archiver/blob_saver.go b/internal/archiver/blob_saver.go index ae4879ff4..d4347a169 100644 --- a/internal/archiver/blob_saver.go +++ b/internal/archiver/blob_saver.go @@ -2,6 +2,7 @@ package archiver import ( "context" + "fmt" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" @@ -43,9 +44,9 @@ func (s *BlobSaver) TriggerShutdown() { // Save stores a blob in the repo. It checks the index and the known blobs // before saving anything. It takes ownership of the buffer passed in. -func (s *BlobSaver) Save(ctx context.Context, t restic.BlobType, buf *Buffer, cb func(res SaveBlobResponse)) { +func (s *BlobSaver) Save(ctx context.Context, t restic.BlobType, buf *Buffer, filename string, cb func(res SaveBlobResponse)) { select { - case s.ch <- saveBlobJob{BlobType: t, buf: buf, cb: cb}: + case s.ch <- saveBlobJob{BlobType: t, buf: buf, fn: filename, cb: cb}: case <-ctx.Done(): debug.Log("not sending job, context is cancelled") } @@ -54,6 +55,7 @@ func (s *BlobSaver) Save(ctx context.Context, t restic.BlobType, buf *Buffer, cb type saveBlobJob struct { restic.BlobType buf *Buffer + fn string cb func(res SaveBlobResponse) } @@ -95,7 +97,7 @@ func (s *BlobSaver) worker(ctx context.Context, jobs <-chan saveBlobJob) error { res, err := s.saveBlob(ctx, job.BlobType, job.buf.Data) if err != nil { debug.Log("saveBlob returned error, exiting: %v", err) - return err + return fmt.Errorf("failed to save blob from file %q: %w", job.fn, err) } job.cb(res) job.buf.Release() diff --git a/internal/archiver/blob_saver_test.go b/internal/archiver/blob_saver_test.go index 1996c35b8..180f95b3d 100644 --- a/internal/archiver/blob_saver_test.go +++ b/internal/archiver/blob_saver_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "runtime" + "strings" "sync" "sync/atomic" "testing" @@ -11,6 +12,7 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" "golang.org/x/sync/errgroup" ) @@ -57,7 +59,7 @@ func TestBlobSaver(t *testing.T) { lock.Lock() results = append(results, SaveBlobResponse{}) lock.Unlock() - b.Save(ctx, restic.DataBlob, buf, func(res SaveBlobResponse) { + b.Save(ctx, restic.DataBlob, buf, "file", func(res SaveBlobResponse) { lock.Lock() results[idx] = res lock.Unlock() @@ -106,7 +108,7 @@ func TestBlobSaverError(t *testing.T) { for i := 0; i < test.blobs; i++ { buf := &Buffer{Data: []byte(fmt.Sprintf("foo%d", i))} - b.Save(ctx, restic.DataBlob, buf, func(res SaveBlobResponse) {}) + b.Save(ctx, restic.DataBlob, buf, "errfile", func(res SaveBlobResponse) {}) } b.TriggerShutdown() @@ -116,9 +118,8 @@ func TestBlobSaverError(t *testing.T) { t.Errorf("expected error not found") } - if err != errTest { - t.Fatalf("unexpected error found: %v", err) - } + rtest.Assert(t, errors.Is(err, errTest), "unexpected error %v", err) + rtest.Assert(t, strings.Contains(err.Error(), "errfile"), "expected error to contain 'errfile' got: %v", err) }) } } diff --git a/internal/archiver/file_saver.go b/internal/archiver/file_saver.go index 724f5e620..7f11bff8a 100644 --- a/internal/archiver/file_saver.go +++ b/internal/archiver/file_saver.go @@ -16,7 +16,7 @@ import ( ) // SaveBlobFn saves a blob to a repo. -type SaveBlobFn func(context.Context, restic.BlobType, *Buffer, func(res SaveBlobResponse)) +type SaveBlobFn func(context.Context, restic.BlobType, *Buffer, string, func(res SaveBlobResponse)) // FileSaver concurrently saves incoming files to the repo. type FileSaver struct { @@ -205,7 +205,7 @@ func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat node.Content = append(node.Content, restic.ID{}) lock.Unlock() - s.saveBlob(ctx, restic.DataBlob, buf, func(sbr SaveBlobResponse) { + s.saveBlob(ctx, restic.DataBlob, buf, target, func(sbr SaveBlobResponse) { lock.Lock() if !sbr.known { fnr.stats.DataBlobs++ diff --git a/internal/archiver/file_saver_test.go b/internal/archiver/file_saver_test.go index b088eeeed..ced9d796e 100644 --- a/internal/archiver/file_saver_test.go +++ b/internal/archiver/file_saver_test.go @@ -33,7 +33,7 @@ func createTestFiles(t testing.TB, num int) (files []string) { func startFileSaver(ctx context.Context, t testing.TB) (*FileSaver, context.Context, *errgroup.Group) { wg, ctx := errgroup.WithContext(ctx) - saveBlob := func(ctx context.Context, tpe restic.BlobType, buf *Buffer, cb func(SaveBlobResponse)) { + saveBlob := func(ctx context.Context, tpe restic.BlobType, buf *Buffer, _ string, cb func(SaveBlobResponse)) { cb(SaveBlobResponse{ id: restic.Hash(buf.Data), length: len(buf.Data), diff --git a/internal/archiver/scanner.go b/internal/archiver/scanner.go index 6ce2a4700..cc419b19e 100644 --- a/internal/archiver/scanner.go +++ b/internal/archiver/scanner.go @@ -25,10 +25,10 @@ type Scanner struct { func NewScanner(fs fs.FS) *Scanner { return &Scanner{ FS: fs, - SelectByName: func(item string) bool { return true }, - Select: func(item string, fi os.FileInfo) bool { return true }, - Error: func(item string, err error) error { return err }, - Result: func(item string, s ScanStats) {}, + SelectByName: func(_ string) bool { return true }, + Select: func(_ string, _ os.FileInfo) bool { return true }, + Error: func(_ string, err error) error { return err }, + Result: func(_ string, _ ScanStats) {}, } } diff --git a/internal/archiver/testing.go b/internal/archiver/testing.go index c7482d160..111c1e68c 100644 --- a/internal/archiver/testing.go +++ b/internal/archiver/testing.go @@ -209,7 +209,7 @@ func TestEnsureFiles(t testing.TB, target string, dir TestDir) { } // TestEnsureFileContent checks if the file in the repo is the same as file. -func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.Repository, filename string, node *restic.Node, file TestFile) { +func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.BlobLoader, filename string, node *restic.Node, file TestFile) { if int(node.Size) != len(file.Content) { t.Fatalf("%v: wrong node size: want %d, got %d", filename, node.Size, len(file.Content)) return @@ -237,7 +237,7 @@ func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.Reposi // TestEnsureTree checks that the tree ID in the repo matches dir. On Windows, // Symlinks are ignored. -func TestEnsureTree(ctx context.Context, t testing.TB, prefix string, repo restic.Repository, treeID restic.ID, dir TestDir) { +func TestEnsureTree(ctx context.Context, t testing.TB, prefix string, repo restic.BlobLoader, treeID restic.ID, dir TestDir) { t.Helper() tree, err := restic.LoadTree(ctx, repo, treeID) diff --git a/internal/archiver/tree_saver.go b/internal/archiver/tree_saver.go index a7dae3873..eae524a78 100644 --- a/internal/archiver/tree_saver.go +++ b/internal/archiver/tree_saver.go @@ -11,7 +11,7 @@ import ( // TreeSaver concurrently saves incoming trees to the repo. type TreeSaver struct { - saveBlob func(ctx context.Context, t restic.BlobType, buf *Buffer, cb func(res SaveBlobResponse)) + saveBlob SaveBlobFn errFn ErrorFunc ch chan<- saveTreeJob @@ -19,7 +19,7 @@ type TreeSaver struct { // NewTreeSaver returns a new tree saver. A worker pool with treeWorkers is // started, it is stopped when ctx is cancelled. -func NewTreeSaver(ctx context.Context, wg *errgroup.Group, treeWorkers uint, saveBlob func(ctx context.Context, t restic.BlobType, buf *Buffer, cb func(res SaveBlobResponse)), errFn ErrorFunc) *TreeSaver { +func NewTreeSaver(ctx context.Context, wg *errgroup.Group, treeWorkers uint, saveBlob SaveBlobFn, errFn ErrorFunc) *TreeSaver { ch := make(chan saveTreeJob) s := &TreeSaver{ @@ -126,7 +126,7 @@ func (s *TreeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, I b := &Buffer{Data: buf} ch := make(chan SaveBlobResponse, 1) - s.saveBlob(ctx, restic.TreeBlob, b, func(res SaveBlobResponse) { + s.saveBlob(ctx, restic.TreeBlob, b, job.target, func(res SaveBlobResponse) { ch <- res }) diff --git a/internal/archiver/tree_saver_test.go b/internal/archiver/tree_saver_test.go index 5de4375d6..47a3f3842 100644 --- a/internal/archiver/tree_saver_test.go +++ b/internal/archiver/tree_saver_test.go @@ -12,7 +12,7 @@ import ( "golang.org/x/sync/errgroup" ) -func treeSaveHelper(_ context.Context, _ restic.BlobType, buf *Buffer, cb func(res SaveBlobResponse)) { +func treeSaveHelper(_ context.Context, _ restic.BlobType, buf *Buffer, _ string, cb func(res SaveBlobResponse)) { cb(SaveBlobResponse{ id: restic.NewRandomID(), known: false, diff --git a/internal/backend/limiter/static_limiter_test.go b/internal/backend/limiter/static_limiter_test.go index 8a839518f..79a1d02f3 100644 --- a/internal/backend/limiter/static_limiter_test.go +++ b/internal/backend/limiter/static_limiter_test.go @@ -118,6 +118,7 @@ func TestRoundTripperReader(t *testing.T) { test.Assert(t, bytes.Equal(data, out.Bytes()), "data ping-pong failed") } +// nolint:bodyclose // the http response is just a mock func TestRoundTripperCornerCases(t *testing.T) { limiter := NewStaticLimiter(Limits{42 * 1024, 42 * 1024}) diff --git a/internal/backend/mem/mem_backend.go b/internal/backend/mem/mem_backend.go index 2698a8275..eea5b060e 100644 --- a/internal/backend/mem/mem_backend.go +++ b/internal/backend/mem/mem_backend.go @@ -28,7 +28,7 @@ func NewFactory() location.Factory { return location.NewHTTPBackendFactory[struct{}, *MemoryBackend]( "mem", - func(s string) (*struct{}, error) { + func(_ string) (*struct{}, error) { return &struct{}{}, nil }, location.NoPassword, diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go index a41a89898..25082598f 100644 --- a/internal/backend/rclone/backend.go +++ b/internal/backend/rclone/backend.go @@ -183,7 +183,7 @@ func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter) (*Backend, dialCount := 0 tr := &http2.Transport{ AllowHTTP: true, // this is not really HTTP, just stdin/stdout - DialTLS: func(network, address string, cfg *tls.Config) (net.Conn, error) { + DialTLS: func(network, address string, _ *tls.Config) (net.Conn, error) { debug.Log("new connection requested, %v %v", network, address) if dialCount > 0 { // the connection to the child process is already closed @@ -252,6 +252,7 @@ func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter) (*Backend, return nil, fmt.Errorf("error talking HTTP to rclone: %w", err) } + _ = res.Body.Close() debug.Log("HTTP status %q returned, moving instance to background", res.Status) err = bg() if err != nil { diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index 5310eba7c..d8171d90e 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -58,6 +58,17 @@ func Open(_ context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) return be, nil } +func drainAndClose(resp *http.Response) error { + _, err := io.Copy(io.Discard, resp.Body) + cerr := resp.Body.Close() + + // return first error + if err != nil { + return errors.Errorf("drain: %w", err) + } + return cerr +} + // Create creates a new REST on server configured in config. func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) { be, err := Open(ctx, cfg, rt) @@ -80,20 +91,14 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, er return nil, err } + if err := drainAndClose(resp); err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("server response unexpected: %v (%v)", resp.Status, resp.StatusCode) } - _, err = io.Copy(io.Discard, resp.Body) - if err != nil { - return nil, err - } - - err = resp.Body.Close() - if err != nil { - return nil, err - } - return be, nil } @@ -136,22 +141,19 @@ func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindR req.ContentLength = rd.Length() resp, err := b.client.Do(req) - - var cerr error - if resp != nil { - _, _ = io.Copy(io.Discard, resp.Body) - cerr = resp.Body.Close() - } - if err != nil { return errors.WithStack(err) } + if err := drainAndClose(resp); err != nil { + return err + } + if resp.StatusCode != http.StatusOK { return errors.Errorf("server response unexpected: %v (%v)", resp.Status, resp.StatusCode) } - return errors.Wrap(cerr, "Close") + return nil } // notExistError is returned whenever the requested file does not exist on the @@ -215,22 +217,17 @@ func (b *Backend) openReader(ctx context.Context, h backend.Handle, length int, req.Header.Set("Accept", ContentTypeV2) resp, err := b.client.Do(req) - if err != nil { - if resp != nil { - _, _ = io.Copy(io.Discard, resp.Body) - _ = resp.Body.Close() - } return nil, errors.Wrap(err, "client.Do") } if resp.StatusCode == http.StatusNotFound { - _ = resp.Body.Close() + _ = drainAndClose(resp) return nil, ¬ExistError{h} } if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { - _ = resp.Body.Close() + _ = drainAndClose(resp) return nil, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status) } @@ -250,13 +247,11 @@ func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, return backend.FileInfo{}, errors.WithStack(err) } - _, _ = io.Copy(io.Discard, resp.Body) - if err = resp.Body.Close(); err != nil { - return backend.FileInfo{}, errors.Wrap(err, "Close") + if err = drainAndClose(resp); err != nil { + return backend.FileInfo{}, err } if resp.StatusCode == http.StatusNotFound { - _ = resp.Body.Close() return backend.FileInfo{}, ¬ExistError{h} } @@ -285,13 +280,15 @@ func (b *Backend) Remove(ctx context.Context, h backend.Handle) error { req.Header.Set("Accept", ContentTypeV2) resp, err := b.client.Do(req) - if err != nil { return errors.Wrap(err, "client.Do") } + if err = drainAndClose(resp); err != nil { + return err + } + if resp.StatusCode == http.StatusNotFound { - _ = resp.Body.Close() return ¬ExistError{h} } @@ -299,12 +296,7 @@ func (b *Backend) Remove(ctx context.Context, h backend.Handle) error { return errors.Errorf("blob not removed, server response: %v (%v)", resp.Status, resp.StatusCode) } - _, err = io.Copy(io.Discard, resp.Body) - if err != nil { - return errors.Wrap(err, "Copy") - } - - return errors.Wrap(resp.Body.Close(), "Close") + return nil } // List runs fn for each file in the backend which has the type t. When an @@ -322,7 +314,6 @@ func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(backend. req.Header.Set("Accept", ContentTypeV2) resp, err := b.client.Do(req) - if err != nil { return errors.Wrap(err, "List") } @@ -333,19 +324,25 @@ func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(backend. // already ignores missing directories, but misuses "not found" to // report certain internal errors, see // https://github.com/rclone/rclone/pull/7550 for details. - return nil + return drainAndClose(resp) } } if resp.StatusCode != http.StatusOK { + _ = drainAndClose(resp) return errors.Errorf("List failed, server response: %v (%v)", resp.Status, resp.StatusCode) } if resp.Header.Get("Content-Type") == ContentTypeV2 { - return b.listv2(ctx, resp, fn) + err = b.listv2(ctx, resp, fn) + } else { + err = b.listv1(ctx, t, resp, fn) } - return b.listv1(ctx, t, resp, fn) + if cerr := drainAndClose(resp); cerr != nil && err == nil { + err = cerr + } + return err } // listv1 uses the REST protocol v1, where a list HTTP request (e.g. `GET diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index f0447224f..d41f4479d 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -325,16 +325,29 @@ func (be *Backend) Path() string { return be.cfg.Prefix } +// useStorageClass returns whether file should be saved in the provided Storage Class +// For archive storage classes, only data files are stored using that class; metadata +// must remain instantly accessible. +func (be *Backend) useStorageClass(h backend.Handle) bool { + notArchiveClass := be.cfg.StorageClass != "GLACIER" && be.cfg.StorageClass != "DEEP_ARCHIVE" + isDataFile := h.Type == backend.PackFile && !h.IsMetadata + return isDataFile || notArchiveClass +} + // Save stores data in the backend at the handle. func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { objName := be.Filename(h) - opts := minio.PutObjectOptions{StorageClass: be.cfg.StorageClass} - opts.ContentType = "application/octet-stream" - // the only option with the high-level api is to let the library handle the checksum computation - opts.SendContentMd5 = true - // only use multipart uploads for very large files - opts.PartSize = 200 * 1024 * 1024 + opts := minio.PutObjectOptions{ + ContentType: "application/octet-stream", + // the only option with the high-level api is to let the library handle the checksum computation + SendContentMd5: true, + // only use multipart uploads for very large files + PartSize: 200 * 1024 * 1024, + } + if be.useStorageClass(h) { + opts.StorageClass = be.cfg.StorageClass + } info, err := be.client.PutObject(ctx, be.cfg.Bucket, objName, io.NopCloser(rd), int64(rd.Length()), opts) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 3bc0fac87..28f55ce3a 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -10,10 +10,10 @@ import ( "sort" "sync" + "github.com/klauspost/compress/zstd" "github.com/minio/sha256-simd" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/s3" - "github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/hashing" @@ -134,7 +134,7 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e if p != nil { var numIndexFiles uint64 - err := indexList.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error { + err := indexList.List(ctx, restic.IndexFile, func(_ restic.ID, _ int64) error { numIndexFiles++ return nil }) @@ -240,17 +240,8 @@ func IsOrphanedPack(err error) bool { } func isS3Legacy(b backend.Backend) bool { - // unwrap cache - if be, ok := b.(*cache.Backend); ok { - b = be.Backend - } - - be, ok := b.(*s3.Backend) - if !ok { - return false - } - - return be.Layout.Name() == "s3legacy" + be := backend.AsBackend[*s3.Backend](b) + return be != nil && be.Layout.Name() == "s3legacy" } // Packs checks that all packs referenced in the index are still available and @@ -361,7 +352,7 @@ func (c *Checker) checkTreeWorker(ctx context.Context, trees <-chan restic.TreeI } } -func loadSnapshotTreeIDs(ctx context.Context, lister restic.Lister, repo restic.Repository) (ids restic.IDs, errs []error) { +func loadSnapshotTreeIDs(ctx context.Context, lister restic.Lister, repo restic.LoaderUnpacked) (ids restic.IDs, errs []error) { err := restic.ForAllSnapshots(ctx, lister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error { if err != nil { errs = append(errs, err) @@ -525,12 +516,20 @@ func (c *Checker) GetPacks() map[restic.ID]int64 { return c.packs } +type partialReadError struct { + err error +} + +func (e *partialReadError) Error() string { + return e.err.Error() +} + // checkPack reads a pack and checks the integrity of all blobs. -func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []restic.Blob, size int64, bufRd *bufio.Reader) error { +func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []restic.Blob, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error { debug.Log("checking pack %v", id.String()) if len(blobs) == 0 { - return errors.Errorf("pack %v is empty or not indexed", id) + return &ErrPackData{PackID: id, errs: []error{errors.New("pack is empty or not indexed")}} } // sanity check blobs in index @@ -551,75 +550,78 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []r var errs []error if nonContinuousPack { debug.Log("Index for pack contains gaps / overlaps, blobs: %v", blobs) - errs = append(errs, errors.New("Index for pack contains gaps / overlapping blobs")) + errs = append(errs, errors.New("index for pack contains gaps / overlapping blobs")) } // calculate hash on-the-fly while reading the pack and capture pack header var hash restic.ID var hdrBuf []byte - hashingLoader := func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - return r.Backend().Load(ctx, h, int(size), 0, func(rd io.Reader) error { - hrd := hashing.NewReader(rd, sha256.New()) - bufRd.Reset(hrd) + h := backend.Handle{Type: backend.PackFile, Name: id.String()} + err := r.Backend().Load(ctx, h, int(size), 0, func(rd io.Reader) error { + hrd := hashing.NewReader(rd, sha256.New()) + bufRd.Reset(hrd) - // skip to start of first blob, offset == 0 for correct pack files - _, err := bufRd.Discard(int(offset)) - if err != nil { - return err + it := repository.NewPackBlobIterator(id, bufRd, 0, blobs, r.Key(), dec) + for { + val, err := it.Next() + if err == repository.ErrPackEOF { + break + } else if err != nil { + return &partialReadError{err} } - - err = fn(bufRd) - if err != nil { - return err + debug.Log(" check blob %v: %v", val.Handle.ID, val.Handle) + if val.Err != nil { + debug.Log(" error verifying blob %v: %v", val.Handle.ID, val.Err) + errs = append(errs, errors.Errorf("blob %v: %v", val.Handle.ID, val.Err)) } - - // skip enough bytes until we reach the possible header start - curPos := length + int(offset) - minHdrStart := int(size) - pack.MaxHeaderSize - if minHdrStart > curPos { - _, err := bufRd.Discard(minHdrStart - curPos) - if err != nil { - return err - } - } - - // read remainder, which should be the pack header - hdrBuf, err = io.ReadAll(bufRd) - if err != nil { - return err - } - - hash = restic.IDFromHash(hrd.Sum(nil)) - return nil - }) - } - - err := repository.StreamPack(ctx, hashingLoader, r.Key(), id, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { - debug.Log(" check blob %v: %v", blob.ID, blob) - if err != nil { - debug.Log(" error verifying blob %v: %v", blob.ID, err) - errs = append(errs, errors.Errorf("blob %v: %v", blob.ID, err)) } + + // skip enough bytes until we reach the possible header start + curPos := lastBlobEnd + minHdrStart := int(size) - pack.MaxHeaderSize + if minHdrStart > curPos { + _, err := bufRd.Discard(minHdrStart - curPos) + if err != nil { + return &partialReadError{err} + } + } + + // read remainder, which should be the pack header + var err error + hdrBuf, err = io.ReadAll(bufRd) + if err != nil { + return &partialReadError{err} + } + + hash = restic.IDFromHash(hrd.Sum(nil)) return nil }) if err != nil { + var e *partialReadError + isPartialReadError := errors.As(err, &e) // failed to load the pack file, return as further checks cannot succeed anyways - debug.Log(" error streaming pack: %v", err) - return errors.Errorf("pack %v failed to download: %v", id, err) + debug.Log(" error streaming pack (partial %v): %v", isPartialReadError, err) + if isPartialReadError { + return &ErrPackData{PackID: id, errs: append(errs, errors.Errorf("partial download error: %w", err))} + } + + // The check command suggests to repair files for which a `ErrPackData` is returned. However, this file + // completely failed to download such that there's no point in repairing anything. + return errors.Errorf("download error: %w", err) } if !hash.Equal(id) { - debug.Log("Pack ID does not match, want %v, got %v", id, hash) - return errors.Errorf("Pack ID does not match, want %v, got %v", id, hash) + debug.Log("pack ID does not match, want %v, got %v", id, hash) + return &ErrPackData{PackID: id, errs: append(errs, errors.Errorf("unexpected pack id %v", hash))} } blobs, hdrSize, err := pack.List(r.Key(), bytes.NewReader(hdrBuf), int64(len(hdrBuf))) if err != nil { - return err + return &ErrPackData{PackID: id, errs: append(errs, err)} } if uint32(idxHdrSize) != hdrSize { debug.Log("Pack header size does not match, want %v, got %v", idxHdrSize, hdrSize) - errs = append(errs, errors.Errorf("Pack header size does not match, want %v, got %v", idxHdrSize, hdrSize)) + errs = append(errs, errors.Errorf("pack header size does not match, want %v, got %v", idxHdrSize, hdrSize)) } idx := r.Index() @@ -633,7 +635,7 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []r } } if !idxHas { - errs = append(errs, errors.Errorf("Blob %v is not contained in index or position is incorrect", blob.ID)) + errs = append(errs, errors.Errorf("blob %v is not contained in index or position is incorrect", blob.ID)) continue } } @@ -670,6 +672,11 @@ func (c *Checker) ReadPacks(ctx context.Context, packs map[restic.ID]int64, p *p // create a buffer that is large enough to be reused by repository.StreamPack // this ensures that we can read the pack header later on bufRd := bufio.NewReaderSize(nil, repository.MaxStreamBufferSize) + dec, err := zstd.NewReader(nil) + if err != nil { + panic(dec) + } + defer dec.Close() for { var ps checkTask var ok bool @@ -683,7 +690,7 @@ func (c *Checker) ReadPacks(ctx context.Context, packs map[restic.ID]int64, p *p } } - err := checkPack(ctx, c.repo, ps.id, ps.blobs, ps.size, bufRd) + err := checkPack(ctx, c.repo, ps.id, ps.blobs, ps.size, bufRd, dec) p.Add(1) if err == nil { continue diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index 752d886e3..0f9179207 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -45,28 +45,6 @@ type EncryptionKey [32]byte type MACKey struct { K [16]byte // for AES-128 R [16]byte // for Poly1305 - - masked bool // remember if the MAC key has already been masked -} - -// mask for key, (cf. http://cr.yp.to/mac/poly1305-20050329.pdf) -var poly1305KeyMask = [16]byte{ - 0xff, - 0xff, - 0xff, - 0x0f, // 3: top four bits zero - 0xfc, // 4: bottom two bits zero - 0xff, - 0xff, - 0x0f, // 7: top four bits zero - 0xfc, // 8: bottom two bits zero - 0xff, - 0xff, - 0x0f, // 11: top four bits zero - 0xfc, // 12: bottom two bits zero - 0xff, - 0xff, - 0x0f, // 15: top four bits zero } func poly1305MAC(msg []byte, nonce []byte, key *MACKey) []byte { @@ -78,32 +56,16 @@ func poly1305MAC(msg []byte, nonce []byte, key *MACKey) []byte { return out[:] } -// mask poly1305 key -func maskKey(k *MACKey) { - if k == nil || k.masked { - return - } - - for i := 0; i < poly1305.TagSize; i++ { - k.R[i] = k.R[i] & poly1305KeyMask[i] - } - - k.masked = true -} - // construct mac key from slice (k||r), with masking func macKeyFromSlice(mk *MACKey, data []byte) { copy(mk.K[:], data[:16]) copy(mk.R[:], data[16:32]) - maskKey(mk) } // prepare key for low-level poly1305.Sum(): r||n func poly1305PrepareKey(nonce []byte, key *MACKey) [32]byte { var k [32]byte - maskKey(key) - cipher, err := aes.NewCipher(key.K[:]) if err != nil { panic(err) @@ -143,7 +105,6 @@ func NewRandomKey() *Key { panic("unable to read enough random bytes for MAC key") } - maskKey(&k.MACKey) return k } diff --git a/internal/dump/common.go b/internal/dump/common.go index c3ba69431..016328835 100644 --- a/internal/dump/common.go +++ b/internal/dump/common.go @@ -16,11 +16,11 @@ import ( type Dumper struct { cache *bloblru.Cache format string - repo restic.Repository + repo restic.BlobLoader w io.Writer } -func New(format string, repo restic.Repository, w io.Writer) *Dumper { +func New(format string, repo restic.BlobLoader, w io.Writer) *Dumper { return &Dumper{ cache: bloblru.New(64 << 20), format: format, @@ -47,7 +47,7 @@ func (d *Dumper) DumpTree(ctx context.Context, tree *restic.Tree, rootPath strin } } -func sendTrees(ctx context.Context, repo restic.Repository, tree *restic.Tree, rootPath string, ch chan *restic.Node) { +func sendTrees(ctx context.Context, repo restic.BlobLoader, tree *restic.Tree, rootPath string, ch chan *restic.Node) { defer close(ch) for _, root := range tree.Nodes { @@ -58,7 +58,7 @@ func sendTrees(ctx context.Context, repo restic.Repository, tree *restic.Tree, r } } -func sendNodes(ctx context.Context, repo restic.Repository, root *restic.Node, ch chan *restic.Node) error { +func sendNodes(ctx context.Context, repo restic.BlobLoader, root *restic.Node, ch chan *restic.Node) error { select { case ch <- root: case <-ctx.Done(): @@ -70,28 +70,28 @@ func sendNodes(ctx context.Context, repo restic.Repository, root *restic.Node, c return nil } - err := walker.Walk(ctx, repo, *root.Subtree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { + err := walker.Walk(ctx, repo, *root.Subtree, walker.WalkVisitor{ProcessNode: func(_ restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { - return false, err + return err } if node == nil { - return false, nil + return nil } node.Path = path.Join(root.Path, nodepath) if !IsFile(node) && !IsDir(node) && !IsLink(node) { - return false, nil + return nil } select { case ch <- node: case <-ctx.Done(): - return false, ctx.Err() + return ctx.Err() } - return false, nil - }) + return nil + }}) return err } diff --git a/internal/errors/errors.go b/internal/errors/errors.go index 0327ea0da..3c669f861 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -2,6 +2,7 @@ package errors import ( stderrors "errors" + "fmt" "github.com/pkg/errors" ) @@ -22,12 +23,42 @@ var Wrap = errors.Wrap // nil, Wrapf returns nil. var Wrapf = errors.Wrapf +// WithStack annotates err with a stack trace at the point WithStack was called. +// If err is nil, WithStack returns nil. var WithStack = errors.WithStack // Go 1.13-style error handling. +// As finds the first error in err's tree that matches target, and if one is found, +// sets target to that error value and returns true. Otherwise, it returns false. func As(err error, tgt interface{}) bool { return stderrors.As(err, tgt) } +// Is reports whether any error in err's tree matches target. func Is(x, y error) bool { return stderrors.Is(x, y) } +// Unwrap returns the result of calling the Unwrap method on err, if err's type contains +// an Unwrap method returning error. Otherwise, Unwrap returns nil. +// +// Unwrap only calls a method of the form "Unwrap() error". In particular Unwrap does not +// unwrap errors returned by [Join]. func Unwrap(err error) error { return stderrors.Unwrap(err) } + +// CombineErrors combines multiple errors into a single error. +func CombineErrors(errors ...error) error { + var combinedErrorMsg string + + for _, err := range errors { + if err != nil { + if combinedErrorMsg != "" { + combinedErrorMsg += "; " // Separate error messages with a delimiter + } + combinedErrorMsg += err.Error() + } + } + + if combinedErrorMsg == "" { + return nil // No errors, return nil + } + + return fmt.Errorf("multiple errors occurred: [%s]", combinedErrorMsg) +} diff --git a/internal/fs/file.go b/internal/fs/file.go index f35901c06..4a236ea09 100644 --- a/internal/fs/file.go +++ b/internal/fs/file.go @@ -124,3 +124,17 @@ func RemoveIfExists(filename string) error { func Chtimes(name string, atime time.Time, mtime time.Time) error { return os.Chtimes(fixpath(name), atime, mtime) } + +// IsAccessDenied checks if the error is due to permission error. +func IsAccessDenied(err error) bool { + return os.IsPermission(err) +} + +// ResetPermissions resets the permissions of the file at the specified path +func ResetPermissions(path string) error { + // Set the default file permissions + if err := os.Chmod(path, 0600); err != nil { + return err + } + return nil +} diff --git a/internal/fs/file_windows.go b/internal/fs/file_windows.go index d19a744e1..2f0969804 100644 --- a/internal/fs/file_windows.go +++ b/internal/fs/file_windows.go @@ -77,3 +77,29 @@ func TempFile(dir, prefix string) (f *os.File, err error) { func Chmod(name string, mode os.FileMode) error { return os.Chmod(fixpath(name), mode) } + +// ClearSystem removes the system attribute from the file. +func ClearSystem(path string) error { + return ClearAttribute(path, windows.FILE_ATTRIBUTE_SYSTEM) +} + +// ClearAttribute removes the specified attribute from the file. +func ClearAttribute(path string, attribute uint32) error { + ptr, err := windows.UTF16PtrFromString(path) + if err != nil { + return err + } + fileAttributes, err := windows.GetFileAttributes(ptr) + if err != nil { + return err + } + if fileAttributes&attribute != 0 { + // Clear the attribute + fileAttributes &= ^uint32(attribute) + err = windows.SetFileAttributes(ptr, fileAttributes) + if err != nil { + return err + } + } + return nil +} diff --git a/internal/fs/fs_track.go b/internal/fs/fs_track.go index 319fbfaff..0c65a8564 100644 --- a/internal/fs/fs_track.go +++ b/internal/fs/fs_track.go @@ -41,7 +41,7 @@ type trackFile struct { func newTrackFile(stack []byte, filename string, file File) *trackFile { f := &trackFile{file} - runtime.SetFinalizer(f, func(f *trackFile) { + runtime.SetFinalizer(f, func(_ *trackFile) { fmt.Fprintf(os.Stderr, "file %s not closed\n\nStacktrack:\n%s\n", filename, stack) panic("file " + filename + " not closed") }) diff --git a/internal/fuse/dir.go b/internal/fuse/dir.go index c5aaf6f52..763a9640c 100644 --- a/internal/fuse/dir.go +++ b/internal/fuse/dir.go @@ -58,7 +58,7 @@ func unwrapCtxCanceled(err error) error { // replaceSpecialNodes replaces nodes with name "." and "/" by their contents. // Otherwise, the node is returned. -func replaceSpecialNodes(ctx context.Context, repo restic.Repository, node *restic.Node) ([]*restic.Node, error) { +func replaceSpecialNodes(ctx context.Context, repo restic.BlobLoader, node *restic.Node) ([]*restic.Node, error) { if node.Type != "dir" || node.Subtree == nil { return []*restic.Node{node}, nil } diff --git a/internal/fuse/fuse_test.go b/internal/fuse/fuse_test.go index 0a121b986..1053d49a4 100644 --- a/internal/fuse/fuse_test.go +++ b/internal/fuse/fuse_test.go @@ -37,7 +37,7 @@ func testRead(t testing.TB, f fs.Handle, offset, length int, data []byte) { rtest.OK(t, fr.Read(ctx, req, resp)) } -func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) { +func firstSnapshotID(t testing.TB, repo restic.Lister) (first restic.ID) { err := repo.List(context.TODO(), restic.SnapshotFile, func(id restic.ID, size int64) error { if first.IsNull() { first = id @@ -52,14 +52,14 @@ func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) { return first } -func loadFirstSnapshot(t testing.TB, repo restic.Repository) *restic.Snapshot { +func loadFirstSnapshot(t testing.TB, repo restic.ListerLoaderUnpacked) *restic.Snapshot { id := firstSnapshotID(t, repo) sn, err := restic.LoadSnapshot(context.TODO(), repo, id) rtest.OK(t, err) return sn } -func loadTree(t testing.TB, repo restic.Repository, id restic.ID) *restic.Tree { +func loadTree(t testing.TB, repo restic.Loader, id restic.ID) *restic.Tree { tree, err := restic.LoadTree(context.TODO(), repo, id) rtest.OK(t, err) return tree diff --git a/internal/fuse/snapshots_dirstruct.go b/internal/fuse/snapshots_dirstruct.go index d40ae6298..049319c6f 100644 --- a/internal/fuse/snapshots_dirstruct.go +++ b/internal/fuse/snapshots_dirstruct.go @@ -295,7 +295,7 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error { } var snapshots restic.Snapshots - err := d.root.cfg.Filter.FindAll(ctx, d.root.repo, d.root.repo, nil, func(id string, sn *restic.Snapshot, err error) error { + err := d.root.cfg.Filter.FindAll(ctx, d.root.repo, d.root.repo, nil, func(_ string, sn *restic.Snapshot, _ error) error { if sn != nil { snapshots = append(snapshots, sn) } diff --git a/internal/index/index_parallel.go b/internal/index/index_parallel.go index d505d756e..d51d5930f 100644 --- a/internal/index/index_parallel.go +++ b/internal/index/index_parallel.go @@ -11,7 +11,7 @@ import ( // ForAllIndexes loads all index files in parallel and calls the given callback. // It is guaranteed that the function is not run concurrently. If the callback // returns an error, this function is cancelled and also returns that error. -func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.Repository, +func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.ListerLoaderUnpacked, fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error { // decoding an index can take quite some time such that this can be both CPU- or IO-bound @@ -19,7 +19,7 @@ func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.Reposi workerCount := repo.Connections() + uint(runtime.GOMAXPROCS(0)) var m sync.Mutex - return restic.ParallelList(ctx, lister, restic.IndexFile, workerCount, func(ctx context.Context, id restic.ID, size int64) error { + return restic.ParallelList(ctx, lister, restic.IndexFile, workerCount, func(ctx context.Context, id restic.ID, _ int64) error { var err error var idx *Index oldFormat := false diff --git a/internal/index/master_index.go b/internal/index/master_index.go index 073c9ace4..4c114b955 100644 --- a/internal/index/master_index.go +++ b/internal/index/master_index.go @@ -9,7 +9,6 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/ui/progress" "golang.org/x/sync/errgroup" ) @@ -267,23 +266,22 @@ func (mi *MasterIndex) MergeFinalIndexes() error { // Save saves all known indexes to index files, leaving out any // packs whose ID is contained in packBlacklist from finalized indexes. -// The new index contains the IDs of all known indexes in the "supersedes" -// field. The IDs are also returned in the IDSet obsolete. -// After calling this function, you should remove the obsolete index files. -func (mi *MasterIndex) Save(ctx context.Context, repo restic.SaverUnpacked, packBlacklist restic.IDSet, extraObsolete restic.IDs, p *progress.Counter) (obsolete restic.IDSet, err error) { - p.SetMax(uint64(len(mi.Packs(packBlacklist)))) +// It also removes the old index files and those listed in extraObsolete. +func (mi *MasterIndex) Save(ctx context.Context, repo restic.Repository, excludePacks restic.IDSet, extraObsolete restic.IDs, opts restic.MasterIndexSaveOpts) error { + p := opts.SaveProgress + p.SetMax(uint64(len(mi.Packs(excludePacks)))) mi.idxMutex.Lock() defer mi.idxMutex.Unlock() - debug.Log("start rebuilding index of %d indexes, pack blacklist: %v", len(mi.idx), packBlacklist) + debug.Log("start rebuilding index of %d indexes, excludePacks: %v", len(mi.idx), excludePacks) newIndex := NewIndex() - obsolete = restic.NewIDSet() + obsolete := restic.NewIDSet() // track spawned goroutines using wg, create a new context which is // cancelled as soon as an error occurs. - wg, ctx := errgroup.WithContext(ctx) + wg, wgCtx := errgroup.WithContext(ctx) ch := make(chan *Index) @@ -310,21 +308,21 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.SaverUnpacked, pack debug.Log("adding index %d", i) - for pbs := range idx.EachByPack(ctx, packBlacklist) { + for pbs := range idx.EachByPack(wgCtx, excludePacks) { newIndex.StorePack(pbs.PackID, pbs.Blobs) p.Add(1) if IndexFull(newIndex, mi.compress) { select { case ch <- newIndex: - case <-ctx.Done(): - return ctx.Err() + case <-wgCtx.Done(): + return wgCtx.Err() } newIndex = NewIndex() } } } - err = newIndex.AddToSupersedes(extraObsolete...) + err := newIndex.AddToSupersedes(extraObsolete...) if err != nil { return err } @@ -332,7 +330,7 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.SaverUnpacked, pack select { case ch <- newIndex: - case <-ctx.Done(): + case <-wgCtx.Done(): } return nil }) @@ -341,7 +339,7 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.SaverUnpacked, pack worker := func() error { for idx := range ch { idx.Finalize() - if _, err := SaveIndex(ctx, repo, idx); err != nil { + if _, err := SaveIndex(wgCtx, repo, idx); err != nil { return err } } @@ -354,9 +352,27 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.SaverUnpacked, pack for i := 0; i < workerCount; i++ { wg.Go(worker) } - err = wg.Wait() + err := wg.Wait() + p.Done() + if err != nil { + return err + } - return obsolete, err + if opts.SkipDeletion { + return nil + } + + p = nil + if opts.DeleteProgress != nil { + p = opts.DeleteProgress() + } + defer p.Done() + return restic.ParallelRemove(ctx, repo, obsolete, restic.IndexFile, func(id restic.ID, err error) error { + if opts.DeleteReport != nil { + opts.DeleteReport(id, err) + } + return err + }, p) } // SaveIndex saves an index in the repository. diff --git a/internal/index/master_index_test.go b/internal/index/master_index_test.go index f76feb5fa..dcf6a94f6 100644 --- a/internal/index/master_index_test.go +++ b/internal/index/master_index_test.go @@ -8,7 +8,6 @@ import ( "testing" "time" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/index" @@ -363,20 +362,11 @@ func testIndexSave(t *testing.T, version uint) { t.Fatal(err) } - obsoletes, err := repo.Index().Save(context.TODO(), repo, nil, nil, nil) + err = repo.Index().Save(context.TODO(), repo, nil, nil, restic.MasterIndexSaveOpts{}) if err != nil { t.Fatalf("unable to save new index: %v", err) } - for id := range obsoletes { - t.Logf("remove index %v", id.Str()) - h := backend.Handle{Type: restic.IndexFile, Name: id.String()} - err = repo.Backend().Remove(context.TODO(), h) - if err != nil { - t.Errorf("error removing index %v: %v", id, err) - } - } - checker := checker.New(repo, false) err = checker.LoadSnapshots(context.TODO()) if err != nil { diff --git a/internal/migrations/upgrade_repo_v2_test.go b/internal/migrations/upgrade_repo_v2_test.go index 40153d3ca..845d20e92 100644 --- a/internal/migrations/upgrade_repo_v2_test.go +++ b/internal/migrations/upgrade_repo_v2_test.go @@ -69,7 +69,7 @@ func TestUpgradeRepoV2Failure(t *testing.T) { Backend: be, } - repo := repository.TestRepositoryWithBackend(t, be, 1) + repo := repository.TestRepositoryWithBackend(t, be, 1, repository.Options{}) if repo.Config().Version != 1 { t.Fatal("test repo has wrong version") } diff --git a/internal/pack/pack.go b/internal/pack/pack.go index 211af7bfb..cd118ab03 100644 --- a/internal/pack/pack.go +++ b/internal/pack/pack.go @@ -1,6 +1,7 @@ package pack import ( + "bytes" "context" "encoding/binary" "fmt" @@ -74,7 +75,7 @@ func (p *Packer) Finalize() error { p.m.Lock() defer p.m.Unlock() - header, err := p.makeHeader() + header, err := makeHeader(p.blobs) if err != nil { return err } @@ -83,6 +84,12 @@ func (p *Packer) Finalize() error { nonce := crypto.NewRandomNonce() encryptedHeader = append(encryptedHeader, nonce...) encryptedHeader = p.k.Seal(encryptedHeader, nonce, header, nil) + encryptedHeader = binary.LittleEndian.AppendUint32(encryptedHeader, uint32(len(encryptedHeader))) + + if err := verifyHeader(p.k, encryptedHeader, p.blobs); err != nil { + //nolint:revive // ignore linter warnings about error message spelling + return fmt.Errorf("Detected data corruption while writing pack-file header: %w\nCorrupted data is either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", err) + } // append the header n, err := p.wr.Write(encryptedHeader) @@ -90,18 +97,33 @@ func (p *Packer) Finalize() error { return errors.Wrap(err, "Write") } - hdrBytes := len(encryptedHeader) - if n != hdrBytes { + if n != len(encryptedHeader) { return errors.New("wrong number of bytes written") } + p.bytes += uint(len(encryptedHeader)) - // write length - err = binary.Write(p.wr, binary.LittleEndian, uint32(hdrBytes)) + return nil +} + +func verifyHeader(k *crypto.Key, header []byte, expected []restic.Blob) error { + // do not offer a way to skip the pack header verification, as pack headers are usually small enough + // to not result in a significant performance impact + + decoded, hdrSize, err := List(k, bytes.NewReader(header), int64(len(header))) if err != nil { - return errors.Wrap(err, "binary.Write") + return fmt.Errorf("header decoding failed: %w", err) + } + if hdrSize != uint32(len(header)) { + return fmt.Errorf("unexpected header size %v instead of %v", hdrSize, len(header)) + } + if len(decoded) != len(expected) { + return fmt.Errorf("pack header size mismatch") + } + for i := 0; i < len(decoded); i++ { + if decoded[i] != expected[i] { + return fmt.Errorf("pack header entry mismatch got %v instead of %v", decoded[i], expected[i]) + } } - p.bytes += uint(hdrBytes + binary.Size(uint32(0))) - return nil } @@ -111,10 +133,10 @@ func (p *Packer) HeaderOverhead() int { } // makeHeader constructs the header for p. -func (p *Packer) makeHeader() ([]byte, error) { - buf := make([]byte, 0, len(p.blobs)*int(entrySize)) +func makeHeader(blobs []restic.Blob) ([]byte, error) { + buf := make([]byte, 0, len(blobs)*int(entrySize)) - for _, b := range p.blobs { + for _, b := range blobs { switch { case b.Type == restic.DataBlob && b.UncompressedLength == 0: buf = append(buf, 0) diff --git a/internal/pack/pack_internal_test.go b/internal/pack/pack_internal_test.go index c1a4867ea..2e7400ad0 100644 --- a/internal/pack/pack_internal_test.go +++ b/internal/pack/pack_internal_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "io" + "strings" "testing" "github.com/restic/restic/internal/crypto" @@ -177,3 +178,60 @@ func TestReadRecords(t *testing.T) { } } } + +func TestUnpackedVerification(t *testing.T) { + // create random keys + k := crypto.NewRandomKey() + blobs := []restic.Blob{ + { + BlobHandle: restic.NewRandomBlobHandle(), + Length: 42, + Offset: 0, + UncompressedLength: 2 * 42, + }, + } + + type DamageType string + const ( + damageData DamageType = "data" + damageCiphertext DamageType = "ciphertext" + damageLength DamageType = "length" + ) + + for _, test := range []struct { + damage DamageType + msg string + }{ + {"", ""}, + {damageData, "pack header entry mismatch"}, + {damageCiphertext, "ciphertext verification failed"}, + {damageLength, "header decoding failed"}, + } { + header, err := makeHeader(blobs) + rtest.OK(t, err) + + if test.damage == damageData { + header[8] ^= 0x42 + } + + encryptedHeader := make([]byte, 0, crypto.CiphertextLength(len(header))) + nonce := crypto.NewRandomNonce() + encryptedHeader = append(encryptedHeader, nonce...) + encryptedHeader = k.Seal(encryptedHeader, nonce, header, nil) + encryptedHeader = binary.LittleEndian.AppendUint32(encryptedHeader, uint32(len(encryptedHeader))) + + if test.damage == damageCiphertext { + encryptedHeader[8] ^= 0x42 + } + if test.damage == damageLength { + encryptedHeader[len(encryptedHeader)-1] ^= 0x42 + } + + err = verifyHeader(k, encryptedHeader, blobs) + if test.msg == "" { + rtest.Assert(t, err == nil, "expected no error, got %v", err) + } else { + rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err) + } + } +} diff --git a/internal/repository/fuzz_test.go b/internal/repository/fuzz_test.go index b4036288c..80372f8e0 100644 --- a/internal/repository/fuzz_test.go +++ b/internal/repository/fuzz_test.go @@ -4,7 +4,6 @@ import ( "context" "testing" - "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/restic" "golang.org/x/sync/errgroup" ) @@ -19,7 +18,7 @@ func FuzzSaveLoadBlob(f *testing.F) { } id := restic.Hash(blob) - repo := TestRepositoryWithBackend(t, mem.New(), 2) + repo := TestRepositoryWithVersion(t, 2) var wg errgroup.Group repo.StartPackUploader(context.TODO(), &wg) diff --git a/internal/repository/key.go b/internal/repository/key.go index 638d15d91..d9f8d8e17 100644 --- a/internal/repository/key.go +++ b/internal/repository/key.go @@ -136,7 +136,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int, defer cancel() // try at most maxKeys keys in repo - err = s.List(listCtx, restic.KeyFile, func(id restic.ID, size int64) error { + err = s.List(listCtx, restic.KeyFile, func(id restic.ID, _ int64) error { checked++ if maxKeys > 0 && checked > maxKeys { return ErrMaxKeysReached @@ -285,6 +285,15 @@ func AddKey(ctx context.Context, s *Repository, password, username, hostname str return newkey, nil } +func RemoveKey(ctx context.Context, repo *Repository, id restic.ID) error { + if id == repo.KeyID() { + return errors.New("refusing to remove key currently used to access repository") + } + + h := backend.Handle{Type: restic.KeyFile, Name: id.String()} + return repo.be.Remove(ctx, h) +} + func (k *Key) String() string { if k == nil { return "" diff --git a/internal/repository/repack.go b/internal/repository/repack.go index c82e63f28..5588984f6 100644 --- a/internal/repository/repack.go +++ b/internal/repository/repack.go @@ -77,7 +77,7 @@ func repack(ctx context.Context, repo restic.Repository, dstRepo restic.Reposito worker := func() error { for t := range downloadQueue { - err := StreamPack(wgCtx, repo.Backend().Load, repo.Key(), t.PackID, t.Blobs, func(blob restic.BlobHandle, buf []byte, err error) error { + err := repo.LoadBlobsFromPack(wgCtx, t.PackID, t.Blobs, func(blob restic.BlobHandle, buf []byte, err error) error { if err != nil { var ierr error // check whether we can get a valid copy somewhere else diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index 20f0f2685..e5e46ac2a 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -62,7 +62,7 @@ func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData fl } } -func createRandomWrongBlob(t testing.TB, repo restic.Repository) { +func createRandomWrongBlob(t testing.TB, repo restic.Repository) restic.BlobHandle { length := randomSize(10*1024, 1024*1024) // 10KiB to 1MiB of data buf := make([]byte, length) rand.Read(buf) @@ -80,6 +80,7 @@ func createRandomWrongBlob(t testing.TB, repo restic.Repository) { if err := repo.Flush(context.Background()); err != nil { t.Fatalf("repo.Flush() returned error %v", err) } + return restic.BlobHandle{ID: id, Type: restic.DataBlob} } // selectBlobs splits the list of all blobs randomly into two lists. A blob @@ -119,7 +120,7 @@ func selectBlobs(t *testing.T, repo restic.Repository, p float32) (list1, list2 return list1, list2 } -func listPacks(t *testing.T, repo restic.Repository) restic.IDSet { +func listPacks(t *testing.T, repo restic.Lister) restic.IDSet { list := restic.NewIDSet() err := repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { list.Insert(id) @@ -173,39 +174,27 @@ func flush(t *testing.T, repo restic.Repository) { func rebuildIndex(t *testing.T, repo restic.Repository) { err := repo.SetIndex(index.NewMasterIndex()) - if err != nil { - t.Fatal(err) - } + rtest.OK(t, err) packs := make(map[restic.ID]int64) err = repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { packs[id] = size return nil }) - if err != nil { - t.Fatal(err) - } + rtest.OK(t, err) _, err = repo.(*repository.Repository).CreateIndexFromPacks(context.TODO(), packs, nil) - if err != nil { - t.Fatal(err) - } + rtest.OK(t, err) + var obsoleteIndexes restic.IDs err = repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error { - h := backend.Handle{ - Type: restic.IndexFile, - Name: id.String(), - } - return repo.Backend().Remove(context.TODO(), h) + obsoleteIndexes = append(obsoleteIndexes, id) + return nil }) - if err != nil { - t.Fatal(err) - } + rtest.OK(t, err) - _, err = repo.Index().Save(context.TODO(), repo, restic.NewIDSet(), nil, nil) - if err != nil { - t.Fatal(err) - } + err = repo.Index().Save(context.TODO(), repo, restic.NewIDSet(), obsoleteIndexes, restic.MasterIndexSaveOpts{}) + rtest.OK(t, err) } func reloadIndex(t *testing.T, repo restic.Repository) { @@ -347,7 +336,8 @@ func TestRepackWrongBlob(t *testing.T) { } func testRepackWrongBlob(t *testing.T, version uint) { - repo := repository.TestRepositoryWithVersion(t, version) + // disable verification to allow adding corrupted blobs to the repository + repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true}) seed := time.Now().UnixNano() rand.Seed(seed) @@ -372,7 +362,8 @@ func TestRepackBlobFallback(t *testing.T) { } func testRepackBlobFallback(t *testing.T, version uint) { - repo := repository.TestRepositoryWithVersion(t, version) + // disable verification to allow adding corrupted blobs to the repository + repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true}) seed := time.Now().UnixNano() rand.Seed(seed) diff --git a/internal/repository/repair_pack.go b/internal/repository/repair_pack.go new file mode 100644 index 000000000..2e0368899 --- /dev/null +++ b/internal/repository/repair_pack.go @@ -0,0 +1,88 @@ +package repository + +import ( + "context" + "errors" + "io" + + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/progress" + "golang.org/x/sync/errgroup" +) + +func RepairPacks(ctx context.Context, repo restic.Repository, ids restic.IDSet, printer progress.Printer) error { + wg, wgCtx := errgroup.WithContext(ctx) + repo.StartPackUploader(wgCtx, wg) + + printer.P("salvaging intact data from specified pack files") + bar := printer.NewCounter("pack files") + bar.SetMax(uint64(len(ids))) + defer bar.Done() + + wg.Go(func() error { + // examine all data the indexes have for the pack file + for b := range repo.Index().ListPacks(wgCtx, ids) { + blobs := b.Blobs + if len(blobs) == 0 { + printer.E("no blobs found for pack %v", b.PackID) + bar.Add(1) + continue + } + + err := repo.LoadBlobsFromPack(wgCtx, b.PackID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { + if err != nil { + // Fallback path + buf, err = repo.LoadBlob(wgCtx, blob.Type, blob.ID, nil) + if err != nil { + printer.E("failed to load blob %v: %v", blob.ID, err) + return nil + } + } + id, _, _, err := repo.SaveBlob(wgCtx, blob.Type, buf, restic.ID{}, true) + if !id.Equal(blob.ID) { + panic("pack id mismatch during upload") + } + return err + }) + // ignore truncated file parts + if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { + return err + } + bar.Add(1) + } + return repo.Flush(wgCtx) + }) + + err := wg.Wait() + bar.Done() + if err != nil { + return err + } + + // remove salvaged packs from index + printer.P("rebuilding index") + + bar = printer.NewCounter("packs processed") + err = repo.Index().Save(ctx, repo, ids, nil, restic.MasterIndexSaveOpts{ + SaveProgress: bar, + DeleteProgress: func() *progress.Counter { + return printer.NewCounter("old indexes deleted") + }, + DeleteReport: func(id restic.ID, _ error) { + printer.VV("removed index %v", id.String()) + }, + }) + + if err != nil { + return err + } + + // cleanup + printer.P("removing salvaged pack files") + // if we fail to delete the damaged pack files, then prune will remove them later on + bar = printer.NewCounter("files deleted") + _ = restic.ParallelRemove(ctx, repo, ids, restic.PackFile, nil, bar) + bar.Done() + + return nil +} diff --git a/internal/repository/repair_pack_test.go b/internal/repository/repair_pack_test.go new file mode 100644 index 000000000..b950245aa --- /dev/null +++ b/internal/repository/repair_pack_test.go @@ -0,0 +1,131 @@ +package repository_test + +import ( + "context" + "math/rand" + "testing" + "time" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/index" + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" + rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui/progress" +) + +func listBlobs(repo restic.Repository) restic.BlobSet { + blobs := restic.NewBlobSet() + repo.Index().Each(context.TODO(), func(pb restic.PackedBlob) { + blobs.Insert(pb.BlobHandle) + }) + return blobs +} + +func replaceFile(t *testing.T, repo restic.Repository, h backend.Handle, damage func([]byte) []byte) { + buf, err := backend.LoadAll(context.TODO(), nil, repo.Backend(), h) + test.OK(t, err) + buf = damage(buf) + test.OK(t, repo.Backend().Remove(context.TODO(), h)) + test.OK(t, repo.Backend().Save(context.TODO(), h, backend.NewByteReader(buf, repo.Backend().Hasher()))) +} + +func TestRepairBrokenPack(t *testing.T) { + repository.TestAllVersions(t, testRepairBrokenPack) +} + +func testRepairBrokenPack(t *testing.T, version uint) { + tests := []struct { + name string + damage func(t *testing.T, repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) + }{ + { + "valid pack", + func(t *testing.T, repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { + return packsBefore, restic.NewBlobSet() + }, + }, + { + "broken pack", + func(t *testing.T, repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { + wrongBlob := createRandomWrongBlob(t, repo) + damagedPacks := findPacksForBlobs(t, repo, restic.NewBlobSet(wrongBlob)) + return damagedPacks, restic.NewBlobSet(wrongBlob) + }, + }, + { + "partially broken pack", + func(t *testing.T, repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { + // damage one of the pack files + damagedID := packsBefore.List()[0] + replaceFile(t, repo, backend.Handle{Type: backend.PackFile, Name: damagedID.String()}, + func(buf []byte) []byte { + buf[0] ^= 0xff + return buf + }) + + // find blob that starts at offset 0 + var damagedBlob restic.BlobHandle + for blobs := range repo.Index().ListPacks(context.TODO(), restic.NewIDSet(damagedID)) { + for _, blob := range blobs.Blobs { + if blob.Offset == 0 { + damagedBlob = blob.BlobHandle + } + } + } + + return restic.NewIDSet(damagedID), restic.NewBlobSet(damagedBlob) + }, + }, { + "truncated pack", + func(t *testing.T, repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { + // damage one of the pack files + damagedID := packsBefore.List()[0] + replaceFile(t, repo, backend.Handle{Type: backend.PackFile, Name: damagedID.String()}, + func(buf []byte) []byte { + buf = buf[0:10] + return buf + }) + + // all blobs in the file are broken + damagedBlobs := restic.NewBlobSet() + for blobs := range repo.Index().ListPacks(context.TODO(), restic.NewIDSet(damagedID)) { + for _, blob := range blobs.Blobs { + damagedBlobs.Insert(blob.BlobHandle) + } + } + return restic.NewIDSet(damagedID), damagedBlobs + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // disable verification to allow adding corrupted blobs to the repository + repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true}) + + seed := time.Now().UnixNano() + rand.Seed(seed) + t.Logf("rand seed is %v", seed) + + createRandomBlobs(t, repo, 5, 0.7) + packsBefore := listPacks(t, repo) + blobsBefore := listBlobs(repo) + + toRepair, damagedBlobs := test.damage(t, repo, packsBefore) + + rtest.OK(t, repository.RepairPacks(context.TODO(), repo, toRepair, &progress.NoopPrinter{})) + // reload index + rtest.OK(t, repo.SetIndex(index.NewMasterIndex())) + rtest.OK(t, repo.LoadIndex(context.TODO(), nil)) + + packsAfter := listPacks(t, repo) + blobsAfter := listBlobs(repo) + + rtest.Assert(t, len(packsAfter.Intersect(toRepair)) == 0, "some damaged packs were not removed") + rtest.Assert(t, len(packsBefore.Sub(toRepair).Sub(packsAfter)) == 0, "not-damaged packs were removed") + rtest.Assert(t, blobsBefore.Sub(damagedBlobs).Equals(blobsAfter), "diverging blob lists") + }) + } +} diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 97dc33fdf..8e34c7125 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -59,8 +59,9 @@ type Repository struct { } type Options struct { - Compression CompressionMode - PackSize uint + Compression CompressionMode + PackSize uint + NoExtraVerify bool } // CompressionMode configures if data should be compressed. @@ -423,6 +424,11 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data // encrypt blob ciphertext = r.key.Seal(ciphertext, nonce, data, nil) + if err := r.verifyCiphertext(ciphertext, uncompressedLength, id); err != nil { + //nolint:revive // ignore linter warnings about error message spelling + return 0, fmt.Errorf("Detected data corruption while saving blob %v: %w\nCorrupted blobs are either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", id, err) + } + // find suitable packer and add blob var pm *packerManager @@ -438,6 +444,31 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data return pm.SaveBlob(ctx, t, id, ciphertext, uncompressedLength) } +func (r *Repository) verifyCiphertext(buf []byte, uncompressedLength int, id restic.ID) error { + if r.opts.NoExtraVerify { + return nil + } + + nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] + plaintext, err := r.key.Open(nil, nonce, ciphertext, nil) + if err != nil { + return fmt.Errorf("decryption failed: %w", err) + } + if uncompressedLength != 0 { + // DecodeAll will allocate a slice if it is not large enough since it + // knows the decompressed size (because we're using EncodeAll) + plaintext, err = r.getZstdDecoder().DecodeAll(plaintext, nil) + if err != nil { + return fmt.Errorf("decompression failed: %w", err) + } + } + if !restic.Hash(plaintext).Equal(id) { + return errors.New("hash mismatch") + } + + return nil +} + func (r *Repository) compressUnpacked(p []byte) ([]byte, error) { // compression is only available starting from version 2 if r.cfg.Version < 2 { @@ -474,7 +505,8 @@ func (r *Repository) decompressUnpacked(p []byte) ([]byte, error) { // SaveUnpacked encrypts data and stores it in the backend. Returned is the // storage hash. -func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []byte) (id restic.ID, err error) { +func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf []byte) (id restic.ID, err error) { + p := buf if t != restic.ConfigFile { p, err = r.compressUnpacked(p) if err != nil { @@ -489,6 +521,11 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by ciphertext = r.key.Seal(ciphertext, nonce, p, nil) + if err := r.verifyUnpacked(ciphertext, t, buf); err != nil { + //nolint:revive // ignore linter warnings about error message spelling + return restic.ID{}, fmt.Errorf("Detected data corruption while saving file of type %v: %w\nCorrupted data is either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", t, err) + } + if t == restic.ConfigFile { id = restic.ID{} } else { @@ -506,6 +543,29 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by return id, nil } +func (r *Repository) verifyUnpacked(buf []byte, t restic.FileType, expected []byte) error { + if r.opts.NoExtraVerify { + return nil + } + + nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] + plaintext, err := r.key.Open(nil, nonce, ciphertext, nil) + if err != nil { + return fmt.Errorf("decryption failed: %w", err) + } + if t != restic.ConfigFile { + plaintext, err = r.decompressUnpacked(plaintext) + if err != nil { + return fmt.Errorf("decompression failed: %w", err) + } + } + + if !bytes.Equal(plaintext, expected) { + return errors.New("data mismatch") + } + return nil +} + // Flush saves all remaining packs and the index func (r *Repository) Flush(ctx context.Context) error { if err := r.flushPacks(ctx); err != nil { @@ -591,7 +651,7 @@ func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error { if p != nil { var numIndexFiles uint64 - err := indexList.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error { + err := indexList.List(ctx, restic.IndexFile, func(_ restic.ID, _ int64) error { numIndexFiles++ return nil }) @@ -602,7 +662,7 @@ func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error { defer p.Done() } - err = index.ForAllIndexes(ctx, indexList, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { + err = index.ForAllIndexes(ctx, indexList, r, func(_ restic.ID, idx *index.Index, _ bool, err error) error { if err != nil { return err } @@ -743,12 +803,19 @@ func (r *Repository) SearchKey(ctx context.Context, password string, maxKeys int return err } + oldKey := r.key + oldKeyID := r.keyID + r.key = key.master r.keyID = key.ID() cfg, err := restic.LoadConfig(ctx, r) - if err == crypto.ErrUnauthenticated { - return fmt.Errorf("config or key %v is damaged: %w", key.ID(), err) - } else if err != nil { + if err != nil { + r.key = oldKey + r.keyID = oldKeyID + + if err == crypto.ErrUnauthenticated { + return fmt.Errorf("config or key %v is damaged: %w", key.ID(), err) + } return fmt.Errorf("config cannot be loaded: %w", err) } @@ -875,16 +942,20 @@ func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte return newID, known, size, err } -type BackendLoadFn func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error +type backendLoadFn func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error // Skip sections with more than 4MB unused blobs const maxUnusedRange = 4 * 1024 * 1024 -// StreamPack loads the listed blobs from the specified pack file. The plaintext blob is passed to +// LoadBlobsFromPack loads the listed blobs from the specified pack file. The plaintext blob is passed to // the handleBlobFn callback or an error if decryption failed or the blob hash does not match. -// handleBlobFn is never called multiple times for the same blob. If the callback returns an error, -// then StreamPack will abort and not retry it. -func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { +// handleBlobFn is called at most once for each blob. If the callback returns an error, +// then LoadBlobsFromPack will abort and not retry it. +func (r *Repository) LoadBlobsFromPack(ctx context.Context, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { + return streamPack(ctx, r.Backend().Load, r.key, packID, blobs, handleBlobFn) +} + +func streamPack(ctx context.Context, beLoad backendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { if len(blobs) == 0 { // nothing to do return nil @@ -915,7 +986,7 @@ func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, pack return streamPackPart(ctx, beLoad, key, packID, blobs[lowerIdx:], handleBlobFn) } -func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { +func streamPackPart(ctx context.Context, beLoad backendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { h := backend.Handle{Type: restic.PackFile, Name: packID.String(), IsMetadata: false} dataStart := blobs[0].Offset @@ -940,72 +1011,18 @@ func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, if bufferSize > MaxStreamBufferSize { bufferSize = MaxStreamBufferSize } - // create reader here to allow reusing the buffered reader from checker.checkData bufRd := bufio.NewReaderSize(rd, bufferSize) - currentBlobEnd := dataStart - var buf []byte - var decode []byte - for len(blobs) > 0 { - entry := blobs[0] + it := NewPackBlobIterator(packID, bufRd, dataStart, blobs, key, dec) - skipBytes := int(entry.Offset - currentBlobEnd) - if skipBytes < 0 { - return errors.Errorf("overlapping blobs in pack %v", packID) - } - - _, err := bufRd.Discard(skipBytes) - if err != nil { + for { + val, err := it.Next() + if err == ErrPackEOF { + break + } else if err != nil { return err } - h := restic.BlobHandle{ID: entry.ID, Type: entry.Type} - debug.Log(" process blob %v, skipped %d, %v", h, skipBytes, entry) - - if uint(cap(buf)) < entry.Length { - buf = make([]byte, entry.Length) - } - buf = buf[:entry.Length] - - n, err := io.ReadFull(bufRd, buf) - if err != nil { - debug.Log(" read error %v", err) - return errors.Wrap(err, "ReadFull") - } - - if n != len(buf) { - return errors.Errorf("read blob %v from %v: not enough bytes read, want %v, got %v", - h, packID.Str(), len(buf), n) - } - currentBlobEnd = entry.Offset + entry.Length - - if int(entry.Length) <= key.NonceSize() { - debug.Log("%v", blobs) - return errors.Errorf("invalid blob length %v", entry) - } - - // decryption errors are likely permanent, give the caller a chance to skip them - nonce, ciphertext := buf[:key.NonceSize()], buf[key.NonceSize():] - plaintext, err := key.Open(ciphertext[:0], nonce, ciphertext, nil) - if err == nil && entry.IsCompressed() { - // DecodeAll will allocate a slice if it is not large enough since it - // knows the decompressed size (because we're using EncodeAll) - decode, err = dec.DecodeAll(plaintext, decode[:0]) - plaintext = decode - if err != nil { - err = errors.Errorf("decompressing blob %v failed: %v", h, err) - } - } - if err == nil { - id := restic.Hash(plaintext) - if !id.Equal(entry.ID) { - debug.Log("read blob %v/%v from %v: wrong data returned, hash is %v", - h.Type, h.ID, packID.Str(), id) - err = errors.Errorf("read blob %v from %v: wrong data returned, hash is %v", - h, packID.Str(), id) - } - } - - err = handleBlobFn(entry.BlobHandle, plaintext, err) + err = handleBlobFn(val.Handle, val.Plaintext, val.Err) if err != nil { cancel() return backoff.Permanent(err) @@ -1018,6 +1035,112 @@ func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, return errors.Wrap(err, "StreamPack") } +type PackBlobIterator struct { + packID restic.ID + rd *bufio.Reader + currentOffset uint + + blobs []restic.Blob + key *crypto.Key + dec *zstd.Decoder + + buf []byte + decode []byte +} + +type PackBlobValue struct { + Handle restic.BlobHandle + Plaintext []byte + Err error +} + +var ErrPackEOF = errors.New("reached EOF of pack file") + +func NewPackBlobIterator(packID restic.ID, rd *bufio.Reader, currentOffset uint, + blobs []restic.Blob, key *crypto.Key, dec *zstd.Decoder) *PackBlobIterator { + return &PackBlobIterator{ + packID: packID, + rd: rd, + currentOffset: currentOffset, + blobs: blobs, + key: key, + dec: dec, + } +} + +// Next returns the next blob, an error or ErrPackEOF if all blobs were read +func (b *PackBlobIterator) Next() (PackBlobValue, error) { + if len(b.blobs) == 0 { + return PackBlobValue{}, ErrPackEOF + } + + entry := b.blobs[0] + b.blobs = b.blobs[1:] + + skipBytes := int(entry.Offset - b.currentOffset) + if skipBytes < 0 { + return PackBlobValue{}, fmt.Errorf("overlapping blobs in pack %v", b.packID) + } + + _, err := b.rd.Discard(skipBytes) + if err != nil { + return PackBlobValue{}, err + } + b.currentOffset = entry.Offset + + h := restic.BlobHandle{ID: entry.ID, Type: entry.Type} + debug.Log(" process blob %v, skipped %d, %v", h, skipBytes, entry) + + if uint(cap(b.buf)) < entry.Length { + b.buf = make([]byte, entry.Length) + } + b.buf = b.buf[:entry.Length] + + n, err := io.ReadFull(b.rd, b.buf) + if err != nil { + debug.Log(" read error %v", err) + return PackBlobValue{}, fmt.Errorf("readFull: %w", err) + } + + if n != len(b.buf) { + return PackBlobValue{}, fmt.Errorf("read blob %v from %v: not enough bytes read, want %v, got %v", + h, b.packID.Str(), len(b.buf), n) + } + b.currentOffset = entry.Offset + entry.Length + + if int(entry.Length) <= b.key.NonceSize() { + debug.Log("%v", b.blobs) + return PackBlobValue{}, fmt.Errorf("invalid blob length %v", entry) + } + + // decryption errors are likely permanent, give the caller a chance to skip them + nonce, ciphertext := b.buf[:b.key.NonceSize()], b.buf[b.key.NonceSize():] + plaintext, err := b.key.Open(ciphertext[:0], nonce, ciphertext, nil) + if err != nil { + err = fmt.Errorf("decrypting blob %v from %v failed: %w", h, b.packID.Str(), err) + } + if err == nil && entry.IsCompressed() { + // DecodeAll will allocate a slice if it is not large enough since it + // knows the decompressed size (because we're using EncodeAll) + b.decode, err = b.dec.DecodeAll(plaintext, b.decode[:0]) + plaintext = b.decode + if err != nil { + err = fmt.Errorf("decompressing blob %v from %v failed: %w", h, b.packID.Str(), err) + } + } + if err == nil { + id := restic.Hash(plaintext) + if !id.Equal(entry.ID) { + debug.Log("read blob %v/%v from %v: wrong data returned, hash is %v", + h.Type, h.ID, b.packID.Str(), id) + err = fmt.Errorf("read blob %v from %v: wrong data returned, hash is %v", + h, b.packID.Str(), id) + } + } + + return PackBlobValue{entry.BlobHandle, plaintext, err}, nil +} + var zeroChunkOnce sync.Once var zeroChunkID restic.ID diff --git a/internal/repository/repository_internal_test.go b/internal/repository/repository_internal_test.go index d8e35b993..0c7115bc9 100644 --- a/internal/repository/repository_internal_test.go +++ b/internal/repository/repository_internal_test.go @@ -1,11 +1,21 @@ package repository import ( + "bytes" + "context" + "encoding/json" + "io" "math/rand" "sort" + "strings" "testing" + "github.com/cenkalti/backoff/v4" + "github.com/google/go-cmp/cmp" + "github.com/klauspost/compress/zstd" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/crypto" + "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -73,3 +83,369 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) { sortCachedPacksFirst(cache, cpy[:]) } } + +// buildPackfileWithoutHeader returns a manually built pack file without a header. +func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool) (blobs []restic.Blob, packfile []byte) { + opts := []zstd.EOption{ + // Set the compression level configured. + zstd.WithEncoderLevel(zstd.SpeedDefault), + // Disable CRC, we have enough checks in place, makes the + // compressed data four bytes shorter. + zstd.WithEncoderCRC(false), + // Set a window of 512kbyte, so we have good lookbehind for usual + // blob sizes. + zstd.WithWindowSize(512 * 1024), + } + enc, err := zstd.NewWriter(nil, opts...) + if err != nil { + panic(err) + } + + var offset uint + for i, size := range blobSizes { + plaintext := rtest.Random(800+i, size) + id := restic.Hash(plaintext) + uncompressedLength := uint(0) + if compress { + uncompressedLength = uint(len(plaintext)) + plaintext = enc.EncodeAll(plaintext, nil) + } + + // we use a deterministic nonce here so the whole process is + // deterministic, last byte is the blob index + var nonce = []byte{ + 0x15, 0x98, 0xc0, 0xf7, 0xb9, 0x65, 0x97, 0x74, + 0x12, 0xdc, 0xd3, 0x62, 0xa9, 0x6e, 0x20, byte(i), + } + + before := len(packfile) + packfile = append(packfile, nonce...) + packfile = key.Seal(packfile, nonce, plaintext, nil) + after := len(packfile) + + ciphertextLength := after - before + + blobs = append(blobs, restic.Blob{ + BlobHandle: restic.BlobHandle{ + Type: restic.DataBlob, + ID: id, + }, + Length: uint(ciphertextLength), + UncompressedLength: uncompressedLength, + Offset: offset, + }) + + offset = uint(len(packfile)) + } + + return blobs, packfile +} + +func TestStreamPack(t *testing.T) { + TestAllVersions(t, testStreamPack) +} + +func testStreamPack(t *testing.T, version uint) { + // always use the same key for deterministic output + const jsonKey = `{"mac":{"k":"eQenuI8adktfzZMuC8rwdA==","r":"k8cfAly2qQSky48CQK7SBA=="},"encrypt":"MKO9gZnRiQFl8mDUurSDa9NMjiu9MUifUrODTHS05wo="}` + + var key crypto.Key + err := json.Unmarshal([]byte(jsonKey), &key) + if err != nil { + t.Fatal(err) + } + + blobSizes := []int{ + 5522811, + 10, + 5231, + 18812, + 123123, + 13522811, + 12301, + 892242, + 28616, + 13351, + 252287, + 188883, + 3522811, + 18883, + } + + var compress bool + switch version { + case 1: + compress = false + case 2: + compress = true + default: + t.Fatal("test does not support repository version", version) + } + + packfileBlobs, packfile := buildPackfileWithoutHeader(blobSizes, &key, compress) + + loadCalls := 0 + shortFirstLoad := false + + loadBytes := func(length int, offset int64) []byte { + data := packfile + + if offset > int64(len(data)) { + offset = 0 + length = 0 + } + data = data[offset:] + + if length > len(data) { + length = len(data) + } + if shortFirstLoad { + length /= 2 + shortFirstLoad = false + } + + return data[:length] + } + + load := func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + data := loadBytes(length, offset) + if shortFirstLoad { + data = data[:len(data)/2] + shortFirstLoad = false + } + + loadCalls++ + + err := fn(bytes.NewReader(data)) + if err == nil { + return nil + } + var permanent *backoff.PermanentError + if errors.As(err, &permanent) { + return err + } + + // retry loading once + return fn(bytes.NewReader(loadBytes(length, offset))) + } + + // first, test regular usage + t.Run("regular", func(t *testing.T) { + tests := []struct { + blobs []restic.Blob + calls int + shortFirstLoad bool + }{ + {packfileBlobs[1:2], 1, false}, + {packfileBlobs[2:5], 1, false}, + {packfileBlobs[2:8], 1, false}, + {[]restic.Blob{ + packfileBlobs[0], + packfileBlobs[4], + packfileBlobs[2], + }, 1, false}, + {[]restic.Blob{ + packfileBlobs[0], + packfileBlobs[len(packfileBlobs)-1], + }, 2, false}, + {packfileBlobs[:], 1, true}, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + gotBlobs := make(map[restic.ID]int) + + handleBlob := func(blob restic.BlobHandle, buf []byte, err error) error { + gotBlobs[blob.ID]++ + + id := restic.Hash(buf) + if !id.Equal(blob.ID) { + t.Fatalf("wrong id %v for blob %s returned", id, blob.ID) + } + + return err + } + + wantBlobs := make(map[restic.ID]int) + for _, blob := range test.blobs { + wantBlobs[blob.ID] = 1 + } + + loadCalls = 0 + shortFirstLoad = test.shortFirstLoad + err = streamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob) + if err != nil { + t.Fatal(err) + } + + if !cmp.Equal(wantBlobs, gotBlobs) { + t.Fatal(cmp.Diff(wantBlobs, gotBlobs)) + } + rtest.Equals(t, test.calls, loadCalls) + }) + } + }) + shortFirstLoad = false + + // next, test invalid uses, which should return an error + t.Run("invalid", func(t *testing.T) { + tests := []struct { + blobs []restic.Blob + err string + }{ + { + // pass one blob several times + blobs: []restic.Blob{ + packfileBlobs[3], + packfileBlobs[8], + packfileBlobs[3], + packfileBlobs[4], + }, + err: "overlapping blobs in pack", + }, + + { + // pass something that's not a valid blob in the current pack file + blobs: []restic.Blob{ + { + Offset: 123, + Length: 20000, + }, + }, + err: "ciphertext verification failed", + }, + + { + // pass a blob that's too small + blobs: []restic.Blob{ + { + Offset: 123, + Length: 10, + }, + }, + err: "invalid blob length", + }, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + handleBlob := func(blob restic.BlobHandle, buf []byte, err error) error { + return err + } + + err = streamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob) + if err == nil { + t.Fatalf("wanted error %v, got nil", test.err) + } + + if !strings.Contains(err.Error(), test.err) { + t.Fatalf("wrong error returned, it should contain %q but was %q", test.err, err) + } + }) + } + }) +} + +func TestBlobVerification(t *testing.T) { + repo := TestRepository(t).(*Repository) + + type DamageType string + const ( + damageData DamageType = "data" + damageCompressed DamageType = "compressed" + damageCiphertext DamageType = "ciphertext" + ) + + for _, test := range []struct { + damage DamageType + msg string + }{ + {"", ""}, + {damageData, "hash mismatch"}, + {damageCompressed, "decompression failed"}, + {damageCiphertext, "ciphertext verification failed"}, + } { + plaintext := rtest.Random(800, 1234) + id := restic.Hash(plaintext) + if test.damage == damageData { + plaintext[42] ^= 0x42 + } + + uncompressedLength := uint(len(plaintext)) + plaintext = repo.getZstdEncoder().EncodeAll(plaintext, nil) + + if test.damage == damageCompressed { + plaintext = plaintext[:len(plaintext)-8] + } + + nonce := crypto.NewRandomNonce() + ciphertext := append([]byte{}, nonce...) + ciphertext = repo.Key().Seal(ciphertext, nonce, plaintext, nil) + + if test.damage == damageCiphertext { + ciphertext[42] ^= 0x42 + } + + err := repo.verifyCiphertext(ciphertext, int(uncompressedLength), id) + if test.msg == "" { + rtest.Assert(t, err == nil, "expected no error, got %v", err) + } else { + rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err) + } + } +} + +func TestUnpackedVerification(t *testing.T) { + repo := TestRepository(t).(*Repository) + + type DamageType string + const ( + damageData DamageType = "data" + damageCompressed DamageType = "compressed" + damageCiphertext DamageType = "ciphertext" + ) + + for _, test := range []struct { + damage DamageType + msg string + }{ + {"", ""}, + {damageData, "data mismatch"}, + {damageCompressed, "decompression failed"}, + {damageCiphertext, "ciphertext verification failed"}, + } { + plaintext := rtest.Random(800, 1234) + orig := append([]byte{}, plaintext...) + if test.damage == damageData { + plaintext[42] ^= 0x42 + } + + compressed := []byte{2} + compressed = repo.getZstdEncoder().EncodeAll(plaintext, compressed) + + if test.damage == damageCompressed { + compressed = compressed[:len(compressed)-8] + } + + nonce := crypto.NewRandomNonce() + ciphertext := append([]byte{}, nonce...) + ciphertext = repo.Key().Seal(ciphertext, nonce, compressed, nil) + + if test.damage == damageCiphertext { + ciphertext[42] ^= 0x42 + } + + err := repo.verifyUnpacked(ciphertext, restic.IndexFile, orig) + if test.msg == "" { + rtest.Assert(t, err == nil, "expected no error, got %v", err) + } else { + rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err) + } + } +} diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 1178a7693..0fa8e4d4a 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -4,8 +4,6 @@ import ( "bytes" "context" "crypto/sha256" - "encoding/json" - "errors" "fmt" "io" "math/rand" @@ -15,9 +13,6 @@ import ( "testing" "time" - "github.com/cenkalti/backoff/v4" - "github.com/google/go-cmp/cmp" - "github.com/klauspost/compress/zstd" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/local" "github.com/restic/restic/internal/crypto" @@ -33,10 +28,19 @@ var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20} var rnd = rand.New(rand.NewSource(time.Now().UnixNano())) func TestSave(t *testing.T) { - repository.TestAllVersions(t, testSave) + repository.TestAllVersions(t, testSavePassID) + repository.TestAllVersions(t, testSaveCalculateID) } -func testSave(t *testing.T, version uint) { +func testSavePassID(t *testing.T, version uint) { + testSave(t, version, false) +} + +func testSaveCalculateID(t *testing.T, version uint) { + testSave(t, version, true) +} + +func testSave(t *testing.T, version uint, calculateID bool) { repo := repository.TestRepositoryWithVersion(t, version) for _, size := range testSizes { @@ -50,51 +54,14 @@ func testSave(t *testing.T, version uint) { repo.StartPackUploader(context.TODO(), &wg) // save - sid, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, restic.ID{}, false) + inputID := restic.ID{} + if !calculateID { + inputID = id + } + sid, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, inputID, false) rtest.OK(t, err) - rtest.Equals(t, id, sid) - rtest.OK(t, repo.Flush(context.Background())) - // rtest.OK(t, repo.SaveIndex()) - - // read back - buf, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, nil) - rtest.OK(t, err) - rtest.Equals(t, size, len(buf)) - - rtest.Assert(t, len(buf) == len(data), - "number of bytes read back does not match: expected %d, got %d", - len(data), len(buf)) - - rtest.Assert(t, bytes.Equal(buf, data), - "data does not match: expected %02x, got %02x", - data, buf) - } -} - -func TestSaveFrom(t *testing.T) { - repository.TestAllVersions(t, testSaveFrom) -} - -func testSaveFrom(t *testing.T, version uint) { - repo := repository.TestRepositoryWithVersion(t, version) - - for _, size := range testSizes { - data := make([]byte, size) - _, err := io.ReadFull(rnd, data) - rtest.OK(t, err) - - id := restic.Hash(data) - - var wg errgroup.Group - repo.StartPackUploader(context.TODO(), &wg) - - // save - id2, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, false) - rtest.OK(t, err) - rtest.Equals(t, id, id2) - rtest.OK(t, repo.Flush(context.Background())) // read back @@ -262,7 +229,7 @@ func TestRepositoryLoadIndex(t *testing.T) { } // loadIndex loads the index id from backend and returns it. -func loadIndex(ctx context.Context, repo restic.Repository, id restic.ID) (*index.Index, error) { +func loadIndex(ctx context.Context, repo restic.LoaderUnpacked, id restic.ID) (*index.Index, error) { buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id) if err != nil { return nil, err @@ -430,274 +397,6 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) { } -// buildPackfileWithoutHeader returns a manually built pack file without a header. -func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool) (blobs []restic.Blob, packfile []byte) { - opts := []zstd.EOption{ - // Set the compression level configured. - zstd.WithEncoderLevel(zstd.SpeedDefault), - // Disable CRC, we have enough checks in place, makes the - // compressed data four bytes shorter. - zstd.WithEncoderCRC(false), - // Set a window of 512kbyte, so we have good lookbehind for usual - // blob sizes. - zstd.WithWindowSize(512 * 1024), - } - enc, err := zstd.NewWriter(nil, opts...) - if err != nil { - panic(err) - } - - var offset uint - for i, size := range blobSizes { - plaintext := rtest.Random(800+i, size) - id := restic.Hash(plaintext) - uncompressedLength := uint(0) - if compress { - uncompressedLength = uint(len(plaintext)) - plaintext = enc.EncodeAll(plaintext, nil) - } - - // we use a deterministic nonce here so the whole process is - // deterministic, last byte is the blob index - var nonce = []byte{ - 0x15, 0x98, 0xc0, 0xf7, 0xb9, 0x65, 0x97, 0x74, - 0x12, 0xdc, 0xd3, 0x62, 0xa9, 0x6e, 0x20, byte(i), - } - - before := len(packfile) - packfile = append(packfile, nonce...) - packfile = key.Seal(packfile, nonce, plaintext, nil) - after := len(packfile) - - ciphertextLength := after - before - - blobs = append(blobs, restic.Blob{ - BlobHandle: restic.BlobHandle{ - Type: restic.DataBlob, - ID: id, - }, - Length: uint(ciphertextLength), - UncompressedLength: uncompressedLength, - Offset: offset, - }) - - offset = uint(len(packfile)) - } - - return blobs, packfile -} - -func TestStreamPack(t *testing.T) { - repository.TestAllVersions(t, testStreamPack) -} - -func testStreamPack(t *testing.T, version uint) { - // always use the same key for deterministic output - const jsonKey = `{"mac":{"k":"eQenuI8adktfzZMuC8rwdA==","r":"k8cfAly2qQSky48CQK7SBA=="},"encrypt":"MKO9gZnRiQFl8mDUurSDa9NMjiu9MUifUrODTHS05wo="}` - - var key crypto.Key - err := json.Unmarshal([]byte(jsonKey), &key) - if err != nil { - t.Fatal(err) - } - - blobSizes := []int{ - 5522811, - 10, - 5231, - 18812, - 123123, - 13522811, - 12301, - 892242, - 28616, - 13351, - 252287, - 188883, - 3522811, - 18883, - } - - var compress bool - switch version { - case 1: - compress = false - case 2: - compress = true - default: - t.Fatal("test does not support repository version", version) - } - - packfileBlobs, packfile := buildPackfileWithoutHeader(blobSizes, &key, compress) - - loadCalls := 0 - shortFirstLoad := false - - loadBytes := func(length int, offset int64) []byte { - data := packfile - - if offset > int64(len(data)) { - offset = 0 - length = 0 - } - data = data[offset:] - - if length > len(data) { - length = len(data) - } - if shortFirstLoad { - length /= 2 - shortFirstLoad = false - } - - return data[:length] - } - - load := func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - data := loadBytes(length, offset) - if shortFirstLoad { - data = data[:len(data)/2] - shortFirstLoad = false - } - - loadCalls++ - - err := fn(bytes.NewReader(data)) - if err == nil { - return nil - } - var permanent *backoff.PermanentError - if errors.As(err, &permanent) { - return err - } - - // retry loading once - return fn(bytes.NewReader(loadBytes(length, offset))) - } - - // first, test regular usage - t.Run("regular", func(t *testing.T) { - tests := []struct { - blobs []restic.Blob - calls int - shortFirstLoad bool - }{ - {packfileBlobs[1:2], 1, false}, - {packfileBlobs[2:5], 1, false}, - {packfileBlobs[2:8], 1, false}, - {[]restic.Blob{ - packfileBlobs[0], - packfileBlobs[4], - packfileBlobs[2], - }, 1, false}, - {[]restic.Blob{ - packfileBlobs[0], - packfileBlobs[len(packfileBlobs)-1], - }, 2, false}, - {packfileBlobs[:], 1, true}, - } - - for _, test := range tests { - t.Run("", func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - gotBlobs := make(map[restic.ID]int) - - handleBlob := func(blob restic.BlobHandle, buf []byte, err error) error { - gotBlobs[blob.ID]++ - - id := restic.Hash(buf) - if !id.Equal(blob.ID) { - t.Fatalf("wrong id %v for blob %s returned", id, blob.ID) - } - - return err - } - - wantBlobs := make(map[restic.ID]int) - for _, blob := range test.blobs { - wantBlobs[blob.ID] = 1 - } - - loadCalls = 0 - shortFirstLoad = test.shortFirstLoad - err = repository.StreamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob) - if err != nil { - t.Fatal(err) - } - - if !cmp.Equal(wantBlobs, gotBlobs) { - t.Fatal(cmp.Diff(wantBlobs, gotBlobs)) - } - rtest.Equals(t, test.calls, loadCalls) - }) - } - }) - shortFirstLoad = false - - // next, test invalid uses, which should return an error - t.Run("invalid", func(t *testing.T) { - tests := []struct { - blobs []restic.Blob - err string - }{ - { - // pass one blob several times - blobs: []restic.Blob{ - packfileBlobs[3], - packfileBlobs[8], - packfileBlobs[3], - packfileBlobs[4], - }, - err: "overlapping blobs in pack", - }, - - { - // pass something that's not a valid blob in the current pack file - blobs: []restic.Blob{ - { - Offset: 123, - Length: 20000, - }, - }, - err: "ciphertext verification failed", - }, - - { - // pass a blob that's too small - blobs: []restic.Blob{ - { - Offset: 123, - Length: 10, - }, - }, - err: "invalid blob length", - }, - } - - for _, test := range tests { - t.Run("", func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - handleBlob := func(blob restic.BlobHandle, buf []byte, err error) error { - return err - } - - err = repository.StreamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob) - if err == nil { - t.Fatalf("wanted error %v, got nil", test.err) - } - - if !strings.Contains(err.Error(), test.err) { - t.Fatalf("wrong error returned, it should contain %q but was %q", test.err, err) - } - }) - } - }) -} - func TestInvalidCompression(t *testing.T) { var comp repository.CompressionMode err := comp.Set("nope") diff --git a/internal/repository/testing.go b/internal/repository/testing.go index d79137425..dbbdbeb07 100644 --- a/internal/repository/testing.go +++ b/internal/repository/testing.go @@ -44,7 +44,7 @@ const TestChunkerPol = chunker.Pol(0x3DA3358B4DC173) // TestRepositoryWithBackend returns a repository initialized with a test // password. If be is nil, an in-memory backend is used. A constant polynomial // is used for the chunker and low-security test parameters. -func TestRepositoryWithBackend(t testing.TB, be backend.Backend, version uint) restic.Repository { +func TestRepositoryWithBackend(t testing.TB, be backend.Backend, version uint, opts Options) restic.Repository { t.Helper() TestUseLowSecurityKDFParameters(t) restic.TestDisableCheckPolynomial(t) @@ -53,7 +53,7 @@ func TestRepositoryWithBackend(t testing.TB, be backend.Backend, version uint) r be = TestBackend(t) } - repo, err := New(be, Options{}) + repo, err := New(be, opts) if err != nil { t.Fatalf("TestRepository(): new repo failed: %v", err) } @@ -79,6 +79,7 @@ func TestRepository(t testing.TB) restic.Repository { func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository { t.Helper() dir := os.Getenv("RESTIC_TEST_REPO") + opts := Options{} if dir != "" { _, err := os.Stat(dir) if err != nil { @@ -86,7 +87,7 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository { if err != nil { t.Fatalf("error creating local backend at %v: %v", dir, err) } - return TestRepositoryWithBackend(t, be, version) + return TestRepositoryWithBackend(t, be, version, opts) } if err == nil { @@ -94,7 +95,7 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository { } } - return TestRepositoryWithBackend(t, nil, version) + return TestRepositoryWithBackend(t, nil, version, opts) } // TestOpenLocal opens a local repository. diff --git a/internal/restic/backend_find.go b/internal/restic/backend_find.go index a6eacabd0..2f00595c4 100644 --- a/internal/restic/backend_find.go +++ b/internal/restic/backend_find.go @@ -30,7 +30,7 @@ func Find(ctx context.Context, be Lister, t FileType, prefix string) (ID, error) ctx, cancel := context.WithCancel(ctx) defer cancel() - err := be.List(ctx, t, func(id ID, size int64) error { + err := be.List(ctx, t, func(id ID, _ int64) error { name := id.String() if len(name) >= len(prefix) && prefix == name[:len(prefix)] { if match.IsNull() { diff --git a/internal/restic/lock.go b/internal/restic/lock.go index 175cf6188..182a3442d 100644 --- a/internal/restic/lock.go +++ b/internal/restic/lock.go @@ -163,9 +163,16 @@ func (l *Lock) fillUserInfo() error { // exclusive lock is found. func (l *Lock) checkForOtherLocks(ctx context.Context) error { var err error + checkedIDs := NewIDSet() + if l.lockID != nil { + checkedIDs.Insert(*l.lockID) + } // retry locking a few times for i := 0; i < 3; i++ { - err = ForAllLocks(ctx, l.repo, l.lockID, func(id ID, lock *Lock, err error) error { + // Store updates in new IDSet to prevent data races + var m sync.Mutex + newCheckedIDs := NewIDSet(checkedIDs.List()...) + err = ForAllLocks(ctx, l.repo, checkedIDs, func(id ID, lock *Lock, err error) error { if err != nil { // if we cannot load a lock then it is unclear whether it can be ignored // it could either be invalid or just unreadable due to network/permission problems @@ -181,8 +188,13 @@ func (l *Lock) checkForOtherLocks(ctx context.Context) error { return &alreadyLockedError{otherLock: lock} } + // valid locks will remain valid + m.Lock() + newCheckedIDs.Insert(id) + m.Unlock() return nil }) + checkedIDs = newCheckedIDs // no lock detected if err == nil { return nil @@ -329,8 +341,8 @@ func (l *Lock) checkExistence(ctx context.Context) (bool, error) { exists := false - err := l.repo.Backend().List(ctx, LockFile, func(fi backend.FileInfo) error { - if fi.Name == l.lockID.String() { + err := l.repo.List(ctx, LockFile, func(id ID, _ int64) error { + if id.Equal(*l.lockID) { exists = true } return nil @@ -367,7 +379,7 @@ func init() { } // LoadLock loads and unserializes a lock from a repository. -func LoadLock(ctx context.Context, repo Repository, id ID) (*Lock, error) { +func LoadLock(ctx context.Context, repo LoaderUnpacked, id ID) (*Lock, error) { lock := &Lock{} if err := LoadJSONUnpacked(ctx, repo, LockFile, id, lock); err != nil { return nil, err @@ -403,7 +415,7 @@ func RemoveStaleLocks(ctx context.Context, repo Repository) (uint, error) { // RemoveAllLocks removes all locks forcefully. func RemoveAllLocks(ctx context.Context, repo Repository) (uint, error) { var processed uint32 - err := ParallelList(ctx, repo, LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error { + err := ParallelList(ctx, repo, LockFile, repo.Connections(), func(ctx context.Context, id ID, _ int64) error { err := repo.Backend().Remove(ctx, backend.Handle{Type: LockFile, Name: id.String()}) if err == nil { atomic.AddUint32(&processed, 1) @@ -417,12 +429,12 @@ func RemoveAllLocks(ctx context.Context, repo Repository) (uint, error) { // It is guaranteed that the function is not run concurrently. If the // callback returns an error, this function is cancelled and also returns that error. // If a lock ID is passed via excludeID, it will be ignored. -func ForAllLocks(ctx context.Context, repo Repository, excludeID *ID, fn func(ID, *Lock, error) error) error { +func ForAllLocks(ctx context.Context, repo ListerLoaderUnpacked, excludeIDs IDSet, fn func(ID, *Lock, error) error) error { var m sync.Mutex // For locks decoding is nearly for free, thus just assume were only limited by IO return ParallelList(ctx, repo, LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error { - if excludeID != nil && id.Equal(*excludeID) { + if excludeIDs.Has(id) { return nil } if size == 0 { diff --git a/internal/restic/lock_test.go b/internal/restic/lock_test.go index faf3f3593..0d282aaf7 100644 --- a/internal/restic/lock_test.go +++ b/internal/restic/lock_test.go @@ -66,7 +66,7 @@ func (be *failLockLoadingBackend) Load(ctx context.Context, h backend.Handle, le func TestMultipleLockFailure(t *testing.T) { be := &failLockLoadingBackend{Backend: mem.New()} - repo := repository.TestRepositoryWithBackend(t, be, 0) + repo := repository.TestRepositoryWithBackend(t, be, 0, repository.Options{}) restic.TestSetLockTimeout(t, 5*time.Millisecond) lock1, err := restic.NewLock(context.TODO(), repo) @@ -120,7 +120,7 @@ func TestExclusiveLockOnLockedRepo(t *testing.T) { rtest.OK(t, elock.Unlock()) } -func createFakeLock(repo restic.Repository, t time.Time, pid int) (restic.ID, error) { +func createFakeLock(repo restic.SaverUnpacked, t time.Time, pid int) (restic.ID, error) { hostname, err := os.Hostname() if err != nil { return restic.ID{}, err @@ -254,7 +254,7 @@ func TestRemoveAllLocks(t *testing.T) { 3, processed) } -func checkSingleLock(t *testing.T, repo restic.Repository) restic.ID { +func checkSingleLock(t *testing.T, repo restic.Lister) restic.ID { t.Helper() var lockID *restic.ID err := repo.List(context.TODO(), restic.LockFile, func(id restic.ID, size int64) error { diff --git a/internal/restic/node.go b/internal/restic/node.go index 7edc41ce8..cbe9ef363 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -6,7 +6,9 @@ import ( "fmt" "os" "os/user" + "reflect" "strconv" + "strings" "sync" "syscall" "time" @@ -20,12 +22,53 @@ import ( "github.com/restic/restic/internal/fs" ) -// ExtendedAttribute is a tuple storing the xattr name and value. +// ExtendedAttribute is a tuple storing the xattr name and value for various filesystems. type ExtendedAttribute struct { Name string `json:"name"` Value []byte `json:"value"` } +// GenericAttributeType can be used for OS specific functionalities by defining specific types +// in node.go to be used by the specific node_xx files. +// OS specific attribute types should follow the convention Attributes. +// GenericAttributeTypes should follow the convention . +// The attributes in OS specific attribute types must be pointers as we want to distinguish nil values +// and not create GenericAttributes for them. +type GenericAttributeType string + +// OSType is the type created to represent each specific OS +type OSType string + +const ( + // When new GenericAttributeType are defined, they must be added in the init function as well. + + // Below are windows specific attributes. + + // TypeCreationTime is the GenericAttributeType used for storing creation time for windows files within the generic attributes map. + TypeCreationTime GenericAttributeType = "windows.creation_time" + // TypeFileAttributes is the GenericAttributeType used for storing file attributes for windows files within the generic attributes map. + TypeFileAttributes GenericAttributeType = "windows.file_attributes" + + // Generic Attributes for other OS types should be defined here. +) + +// init is called when the package is initialized. Any new GenericAttributeTypes being created must be added here as well. +func init() { + storeGenericAttributeType(TypeCreationTime, TypeFileAttributes) +} + +// genericAttributesForOS maintains a map of known genericAttributesForOS to the OSType +var genericAttributesForOS = map[GenericAttributeType]OSType{} + +// storeGenericAttributeType adds and entry in genericAttributesForOS map +func storeGenericAttributeType(attributeTypes ...GenericAttributeType) { + for _, attributeType := range attributeTypes { + // Get the OS attribute type from the GenericAttributeType + osAttributeName := strings.Split(string(attributeType), ".")[0] + genericAttributesForOS[attributeType] = OSType(osAttributeName) + } +} + // Node is a file, directory or other item in a backup. type Node struct { Name string `json:"name"` @@ -47,11 +90,12 @@ type Node struct { // This allows storing arbitrary byte-sequences, which are possible as symlink targets on unix systems, // as LinkTarget without breaking backwards-compatibility. // Must only be set of the linktarget cannot be encoded as valid utf8. - LinkTargetRaw []byte `json:"linktarget_raw,omitempty"` - ExtendedAttributes []ExtendedAttribute `json:"extended_attributes,omitempty"` - Device uint64 `json:"device,omitempty"` // in case of Type == "dev", stat.st_rdev - Content IDs `json:"content"` - Subtree *ID `json:"subtree,omitempty"` + LinkTargetRaw []byte `json:"linktarget_raw,omitempty"` + ExtendedAttributes []ExtendedAttribute `json:"extended_attributes,omitempty"` + GenericAttributes map[GenericAttributeType]json.RawMessage `json:"generic_attributes,omitempty"` + Device uint64 `json:"device,omitempty"` // in case of Type == "dev", stat.st_rdev + Content IDs `json:"content"` + Subtree *ID `json:"subtree,omitempty"` Error string `json:"error,omitempty"` @@ -142,7 +186,7 @@ func (node Node) GetExtendedAttribute(a string) []byte { } // CreateAt creates the node at the given path but does NOT restore node meta data. -func (node *Node) CreateAt(ctx context.Context, path string, repo Repository) error { +func (node *Node) CreateAt(ctx context.Context, path string, repo BlobLoader) error { debug.Log("create node %v at %v", node.Name, path) switch node.Type { @@ -180,8 +224,8 @@ func (node *Node) CreateAt(ctx context.Context, path string, repo Repository) er } // RestoreMetadata restores node metadata -func (node Node) RestoreMetadata(path string) error { - err := node.restoreMetadata(path) +func (node Node) RestoreMetadata(path string, warn func(msg string)) error { + err := node.restoreMetadata(path, warn) if err != nil { debug.Log("restoreMetadata(%s) error %v", path, err) } @@ -189,7 +233,7 @@ func (node Node) RestoreMetadata(path string) error { return err } -func (node Node) restoreMetadata(path string) error { +func (node Node) restoreMetadata(path string, warn func(msg string)) error { var firsterr error if err := lchown(path, int(node.UID), int(node.GID)); err != nil { @@ -203,14 +247,6 @@ func (node Node) restoreMetadata(path string) error { } } - if node.Type != "symlink" { - if err := fs.Chmod(path, node.Mode); err != nil { - if firsterr != nil { - firsterr = errors.WithStack(err) - } - } - } - if err := node.RestoreTimestamps(path); err != nil { debug.Log("error restoring timestamps for dir %v: %v", path, err) if firsterr != nil { @@ -225,6 +261,24 @@ func (node Node) restoreMetadata(path string) error { } } + if err := node.restoreGenericAttributes(path, warn); err != nil { + debug.Log("error restoring generic attributes for %v: %v", path, err) + if firsterr != nil { + firsterr = err + } + } + + // Moving RestoreTimestamps and restoreExtendedAttributes calls above as for readonly files in windows + // calling Chmod below will no longer allow any modifications to be made on the file and the + // calls above would fail. + if node.Type != "symlink" { + if err := fs.Chmod(path, node.Mode); err != nil { + if firsterr != nil { + firsterr = errors.WithStack(err) + } + } + } + return firsterr } @@ -264,7 +318,7 @@ func (node Node) createDirAt(path string) error { return nil } -func (node Node) createFileAt(ctx context.Context, path string, repo Repository) error { +func (node Node) createFileAt(ctx context.Context, path string, repo BlobLoader) error { f, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) if err != nil { return errors.WithStack(err) @@ -284,7 +338,7 @@ func (node Node) createFileAt(ctx context.Context, path string, repo Repository) return nil } -func (node Node) writeNodeContent(ctx context.Context, repo Repository, f *os.File) error { +func (node Node) writeNodeContent(ctx context.Context, repo BlobLoader, f *os.File) error { var buf []byte for _, id := range node.Content { buf, err := repo.LoadBlob(ctx, DataBlob, id, buf) @@ -438,6 +492,9 @@ func (node Node) Equals(other Node) bool { if !node.sameExtendedAttributes(other) { return false } + if !node.sameGenericAttributes(other) { + return false + } if node.Subtree != nil { if other.Subtree == nil { return false @@ -480,8 +537,13 @@ func (node Node) sameContent(other Node) bool { } func (node Node) sameExtendedAttributes(other Node) bool { - if len(node.ExtendedAttributes) != len(other.ExtendedAttributes) { + ln := len(node.ExtendedAttributes) + lo := len(other.ExtendedAttributes) + if ln != lo { return false + } else if ln == 0 { + // This means lo is also of length 0 + return true } // build a set of all attributes that node has @@ -525,6 +587,33 @@ func (node Node) sameExtendedAttributes(other Node) bool { return true } +func (node Node) sameGenericAttributes(other Node) bool { + return deepEqual(node.GenericAttributes, other.GenericAttributes) +} + +func deepEqual(map1, map2 map[GenericAttributeType]json.RawMessage) bool { + // Check if the maps have the same number of keys + if len(map1) != len(map2) { + return false + } + + // Iterate over each key-value pair in map1 + for key, value1 := range map1 { + // Check if the key exists in map2 + value2, ok := map2[key] + if !ok { + return false + } + + // Check if the JSON.RawMessage values are equal byte by byte + if !bytes.Equal(value1, value2) { + return false + } + } + + return true +} + func (node *Node) fillUser(stat *statT) { uid, gid := stat.uid(), stat.gid() node.UID, node.GID = uid, gid @@ -627,7 +716,17 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error { return errors.Errorf("unsupported file type %q", node.Type) } - return node.fillExtendedAttributes(path) + allowExtended, err := node.fillGenericAttributes(path, fi, stat) + if allowExtended { + // Skip processing ExtendedAttributes if allowExtended is false. + errEx := node.fillExtendedAttributes(path) + if err == nil { + err = errEx + } else { + debug.Log("Error filling extended attributes for %v at %v : %v", node.Name, path, errEx) + } + } + return err } func (node *Node) fillExtendedAttributes(path string) error { @@ -665,3 +764,119 @@ func (node *Node) fillTimes(stat *statT) { node.ChangeTime = time.Unix(ctim.Unix()) node.AccessTime = time.Unix(atim.Unix()) } + +// HandleUnknownGenericAttributesFound is used for handling and distinguing between scenarios related to future versions and cross-OS repositories +func HandleUnknownGenericAttributesFound(unknownAttribs []GenericAttributeType, warn func(msg string)) { + for _, unknownAttrib := range unknownAttribs { + handleUnknownGenericAttributeFound(unknownAttrib, warn) + } +} + +// handleUnknownGenericAttributeFound is used for handling and distinguing between scenarios related to future versions and cross-OS repositories +func handleUnknownGenericAttributeFound(genericAttributeType GenericAttributeType, warn func(msg string)) { + if checkGenericAttributeNameNotHandledAndPut(genericAttributeType) { + // Print the unique error only once for a given execution + os, exists := genericAttributesForOS[genericAttributeType] + + if exists { + // If genericAttributesForOS contains an entry but we still got here, it means the specific node_xx.go for the current OS did not handle it and the repository may have been originally created on a different OS. + // The fact that node.go knows about the attribute, means it is not a new attribute. This may be a common situation if a repo is used across OSs. + debug.Log("Ignoring a generic attribute found in the repository: %s which may not be compatible with your OS. Compatible OS: %s", genericAttributeType, os) + } else { + // If genericAttributesForOS in node.go does not know about this attribute, then the repository may have been created by a newer version which has a newer GenericAttributeType. + warn(fmt.Sprintf("Found an unrecognized generic attribute in the repository: %s. You may need to upgrade to latest version of restic.", genericAttributeType)) + } + } +} + +// handleAllUnknownGenericAttributesFound performs validations for all generic attributes in the node. +// This is not used on windows currently because windows has handling for generic attributes. +// nolint:unused +func (node Node) handleAllUnknownGenericAttributesFound(warn func(msg string)) error { + for name := range node.GenericAttributes { + handleUnknownGenericAttributeFound(name, warn) + } + return nil +} + +var unknownGenericAttributesHandlingHistory sync.Map + +// checkGenericAttributeNameNotHandledAndPut checks if the GenericAttributeType name entry +// already exists and puts it in the map if not. +func checkGenericAttributeNameNotHandledAndPut(value GenericAttributeType) bool { + // If Key doesn't exist, put the value and return true because it is not already handled + _, exists := unknownGenericAttributesHandlingHistory.LoadOrStore(value, "") + // Key exists, then it is already handled so return false + return !exists +} + +// The functions below are common helper functions which can be used for generic attributes support +// across different OS. + +// genericAttributesToOSAttrs gets the os specific attribute from the generic attribute using reflection +// nolint:unused +func genericAttributesToOSAttrs(attrs map[GenericAttributeType]json.RawMessage, attributeType reflect.Type, attributeValuePtr *reflect.Value, keyPrefix string) (unknownAttribs []GenericAttributeType, err error) { + attributeValue := *attributeValuePtr + + for key, rawMsg := range attrs { + found := false + for i := 0; i < attributeType.NumField(); i++ { + if getFQKeyByIndex(attributeType, i, keyPrefix) == key { + found = true + fieldValue := attributeValue.Field(i) + // For directly supported types, use json.Unmarshal directly + if err := json.Unmarshal(rawMsg, fieldValue.Addr().Interface()); err != nil { + return unknownAttribs, errors.Wrap(err, "Unmarshal") + } + break + } + } + if !found { + unknownAttribs = append(unknownAttribs, key) + } + } + return unknownAttribs, nil +} + +// getFQKey gets the fully qualified key for the field +// nolint:unused +func getFQKey(field reflect.StructField, keyPrefix string) GenericAttributeType { + return GenericAttributeType(fmt.Sprintf("%s.%s", keyPrefix, field.Tag.Get("generic"))) +} + +// getFQKeyByIndex gets the fully qualified key for the field index +// nolint:unused +func getFQKeyByIndex(attributeType reflect.Type, index int, keyPrefix string) GenericAttributeType { + return getFQKey(attributeType.Field(index), keyPrefix) +} + +// osAttrsToGenericAttributes gets the generic attribute from the os specific attribute using reflection +// nolint:unused +func osAttrsToGenericAttributes(attributeType reflect.Type, attributeValuePtr *reflect.Value, keyPrefix string) (attrs map[GenericAttributeType]json.RawMessage, err error) { + attributeValue := *attributeValuePtr + attrs = make(map[GenericAttributeType]json.RawMessage) + + // Iterate over the fields of the struct + for i := 0; i < attributeType.NumField(); i++ { + field := attributeType.Field(i) + + // Get the field value using reflection + fieldValue := attributeValue.FieldByName(field.Name) + + // Check if the field is nil + if fieldValue.IsNil() { + // If it's nil, skip this field + continue + } + + // Marshal the field value into a json.RawMessage + var fieldBytes []byte + if fieldBytes, err = json.Marshal(fieldValue.Interface()); err != nil { + return attrs, errors.Wrap(err, "Marshal") + } + + // Insert the field into the map + attrs[getFQKey(field, keyPrefix)] = json.RawMessage(fieldBytes) + } + return attrs, nil +} diff --git a/internal/restic/node_aix.go b/internal/restic/node_aix.go index 572e33a65..def46bd60 100644 --- a/internal/restic/node_aix.go +++ b/internal/restic/node_aix.go @@ -3,9 +3,12 @@ package restic -import "syscall" +import ( + "os" + "syscall" +) -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { return nil } @@ -34,3 +37,13 @@ func Listxattr(path string) ([]string, error) { func Setxattr(path, name string, data []byte) error { return nil } + +// restoreGenericAttributes is no-op on AIX. +func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { + return node.handleAllUnknownGenericAttributesFound(warn) +} + +// fillGenericAttributes is a no-op on AIX. +func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { + return true, nil +} diff --git a/internal/restic/node_netbsd.go b/internal/restic/node_netbsd.go index 0eade2f37..1a47299be 100644 --- a/internal/restic/node_netbsd.go +++ b/internal/restic/node_netbsd.go @@ -1,8 +1,11 @@ package restic -import "syscall" +import ( + "os" + "syscall" +) -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { return nil } @@ -10,18 +13,27 @@ func (s statT) atim() syscall.Timespec { return s.Atimespec } func (s statT) mtim() syscall.Timespec { return s.Mtimespec } func (s statT) ctim() syscall.Timespec { return s.Ctimespec } -// Getxattr retrieves extended attribute data associated with path. +// Getxattr is a no-op on netbsd. func Getxattr(path, name string) ([]byte, error) { return nil, nil } -// Listxattr retrieves a list of names of extended attributes associated with the -// given path in the file system. +// Listxattr is a no-op on netbsd. func Listxattr(path string) ([]string, error) { return nil, nil } -// Setxattr associates name and data together as an attribute of path. +// Setxattr is a no-op on netbsd. func Setxattr(path, name string, data []byte) error { return nil } + +// restoreGenericAttributes is no-op on netbsd. +func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { + return node.handleAllUnknownGenericAttributesFound(warn) +} + +// fillGenericAttributes is a no-op on netbsd. +func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { + return true, nil +} diff --git a/internal/restic/node_openbsd.go b/internal/restic/node_openbsd.go index a4ccc7211..e60eb9dc8 100644 --- a/internal/restic/node_openbsd.go +++ b/internal/restic/node_openbsd.go @@ -1,8 +1,11 @@ package restic -import "syscall" +import ( + "os" + "syscall" +) -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { return nil } @@ -10,18 +13,27 @@ func (s statT) atim() syscall.Timespec { return s.Atim } func (s statT) mtim() syscall.Timespec { return s.Mtim } func (s statT) ctim() syscall.Timespec { return s.Ctim } -// Getxattr retrieves extended attribute data associated with path. +// Getxattr is a no-op on openbsd. func Getxattr(path, name string) ([]byte, error) { return nil, nil } -// Listxattr retrieves a list of names of extended attributes associated with the -// given path in the file system. +// Listxattr is a no-op on openbsd. func Listxattr(path string) ([]string, error) { return nil, nil } -// Setxattr associates name and data together as an attribute of path. +// Setxattr is a no-op on openbsd. func Setxattr(path, name string, data []byte) error { return nil } + +// restoreGenericAttributes is no-op on openbsd. +func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { + return node.handleAllUnknownGenericAttributesFound(warn) +} + +// fillGenericAttributes is a no-op on openbsd. +func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { + return true, nil +} diff --git a/internal/restic/node_test.go b/internal/restic/node_test.go index aae010421..d9fa02ac8 100644 --- a/internal/restic/node_test.go +++ b/internal/restic/node_test.go @@ -1,4 +1,4 @@ -package restic_test +package restic import ( "context" @@ -11,7 +11,6 @@ import ( "testing" "time" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test" ) @@ -32,7 +31,7 @@ func BenchmarkNodeFillUser(t *testing.B) { t.ResetTimer() for i := 0; i < t.N; i++ { - _, err := restic.NodeFromFileInfo(path, fi) + _, err := NodeFromFileInfo(path, fi) rtest.OK(t, err) } @@ -56,7 +55,7 @@ func BenchmarkNodeFromFileInfo(t *testing.B) { t.ResetTimer() for i := 0; i < t.N; i++ { - _, err := restic.NodeFromFileInfo(path, fi) + _, err := NodeFromFileInfo(path, fi) if err != nil { t.Fatal(err) } @@ -75,11 +74,11 @@ func parseTime(s string) time.Time { return t.Local() } -var nodeTests = []restic.Node{ +var nodeTests = []Node{ { Name: "testFile", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0604, @@ -90,7 +89,7 @@ var nodeTests = []restic.Node{ { Name: "testSuidFile", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0755 | os.ModeSetuid, @@ -101,7 +100,7 @@ var nodeTests = []restic.Node{ { Name: "testSuidFile2", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0755 | os.ModeSetgid, @@ -112,7 +111,7 @@ var nodeTests = []restic.Node{ { Name: "testSticky", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0755 | os.ModeSticky, @@ -148,7 +147,7 @@ var nodeTests = []restic.Node{ { Name: "testFile", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0604, @@ -170,14 +169,14 @@ var nodeTests = []restic.Node{ { Name: "testXattrFile", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0604, ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []restic.ExtendedAttribute{ + ExtendedAttributes: []ExtendedAttribute{ {"user.foo", []byte("bar")}, }, }, @@ -191,7 +190,7 @@ var nodeTests = []restic.Node{ ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []restic.ExtendedAttribute{ + ExtendedAttributes: []ExtendedAttribute{ {"user.foo", []byte("bar")}, }, }, @@ -219,7 +218,7 @@ func TestNodeRestoreAt(t *testing.T) { nodePath = filepath.Join(tempdir, test.Name) } rtest.OK(t, test.CreateAt(context.TODO(), nodePath, nil)) - rtest.OK(t, test.RestoreMetadata(nodePath)) + rtest.OK(t, test.RestoreMetadata(nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) })) if test.Type == "dir" { rtest.OK(t, test.RestoreTimestamps(nodePath)) @@ -228,7 +227,7 @@ func TestNodeRestoreAt(t *testing.T) { fi, err := os.Lstat(nodePath) rtest.OK(t, err) - n2, err := restic.NodeFromFileInfo(nodePath, fi) + n2, err := NodeFromFileInfo(nodePath, fi) rtest.OK(t, err) rtest.Assert(t, test.Name == n2.Name, @@ -330,7 +329,7 @@ func TestFixTime(t *testing.T) { for _, test := range tests { t.Run("", func(t *testing.T) { - res := restic.FixTime(test.src) + res := FixTime(test.src) if !res.Equal(test.want) { t.Fatalf("wrong result for %v, want:\n %v\ngot:\n %v", test.src, test.want, res) } @@ -343,12 +342,12 @@ func TestSymlinkSerialization(t *testing.T) { "válîd \t Üñi¢òde \n śẗŕinǵ", string([]byte{0, 1, 2, 0xfa, 0xfb, 0xfc}), } { - n := restic.Node{ + n := Node{ LinkTarget: link, } ser, err := json.Marshal(n) test.OK(t, err) - var n2 restic.Node + var n2 Node err = json.Unmarshal(ser, &n2) test.OK(t, err) fmt.Println(string(ser)) @@ -365,7 +364,7 @@ func TestSymlinkSerializationFormat(t *testing.T) { {`{"linktarget":"test"}`, "test"}, {`{"linktarget":"\u0000\u0001\u0002\ufffd\ufffd\ufffd","linktarget_raw":"AAEC+vv8"}`, string([]byte{0, 1, 2, 0xfa, 0xfb, 0xfc})}, } { - var n2 restic.Node + var n2 Node err := json.Unmarshal([]byte(d.ser), &n2) test.OK(t, err) test.Equals(t, d.linkTarget, n2.LinkTarget) diff --git a/internal/restic/node_windows.go b/internal/restic/node_windows.go index fc6439b40..5875c3ccd 100644 --- a/internal/restic/node_windows.go +++ b/internal/restic/node_windows.go @@ -1,21 +1,47 @@ package restic import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" "syscall" + "unsafe" + "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/fs" + "golang.org/x/sys/windows" +) + +// WindowsAttributes are the genericAttributes for Windows OS +type WindowsAttributes struct { + // CreationTime is used for storing creation time for windows files. + CreationTime *syscall.Filetime `generic:"creation_time"` + // FileAttributes is used for storing file attributes for windows files. + FileAttributes *uint32 `generic:"file_attributes"` +} + +var ( + modAdvapi32 = syscall.NewLazyDLL("advapi32.dll") + procEncryptFile = modAdvapi32.NewProc("EncryptFileW") + procDecryptFile = modAdvapi32.NewProc("DecryptFileW") ) // mknod is not supported on Windows. -func mknod(path string, mode uint32, dev uint64) (err error) { +func mknod(_ string, mode uint32, dev uint64) (err error) { return errors.New("device nodes cannot be created on windows") } // Windows doesn't need lchown -func lchown(path string, uid int, gid int) (err error) { +func lchown(_ string, uid int, gid int) (err error) { return nil } +// restoreSymlinkTimestamps restores timestamps for symlinks func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { // tweaked version of UtimesNano from go/src/syscall/syscall_windows.go pathp, e := syscall.UTF16PtrFromString(path) @@ -28,7 +54,14 @@ func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespe if e != nil { return e } - defer syscall.Close(h) + + defer func() { + err := syscall.Close(h) + if err != nil { + debug.Log("Error closing file handle for %s: %v\n", path, err) + } + }() + a := syscall.NsecToFiletime(syscall.TimespecToNsec(utimes[0])) w := syscall.NsecToFiletime(syscall.TimespecToNsec(utimes[1])) return syscall.SetFileTime(h, nil, &a, &w) @@ -83,3 +116,188 @@ func (s statT) ctim() syscall.Timespec { // Windows does not have the concept of a "change time" in the sense Unix uses it, so we're using the LastWriteTime here. return syscall.NsecToTimespec(s.LastWriteTime.Nanoseconds()) } + +// restoreGenericAttributes restores generic attributes for Windows +func (node Node) restoreGenericAttributes(path string, warn func(msg string)) (err error) { + if len(node.GenericAttributes) == 0 { + return nil + } + var errs []error + windowsAttributes, unknownAttribs, err := genericAttributesToWindowsAttrs(node.GenericAttributes) + if err != nil { + return fmt.Errorf("error parsing generic attribute for: %s : %v", path, err) + } + if windowsAttributes.CreationTime != nil { + if err := restoreCreationTime(path, windowsAttributes.CreationTime); err != nil { + errs = append(errs, fmt.Errorf("error restoring creation time for: %s : %v", path, err)) + } + } + if windowsAttributes.FileAttributes != nil { + if err := restoreFileAttributes(path, windowsAttributes.FileAttributes); err != nil { + errs = append(errs, fmt.Errorf("error restoring file attributes for: %s : %v", path, err)) + } + } + + HandleUnknownGenericAttributesFound(unknownAttribs, warn) + return errors.CombineErrors(errs...) +} + +// genericAttributesToWindowsAttrs converts the generic attributes map to a WindowsAttributes and also returns a string of unkown attributes that it could not convert. +func genericAttributesToWindowsAttrs(attrs map[GenericAttributeType]json.RawMessage) (windowsAttributes WindowsAttributes, unknownAttribs []GenericAttributeType, err error) { + waValue := reflect.ValueOf(&windowsAttributes).Elem() + unknownAttribs, err = genericAttributesToOSAttrs(attrs, reflect.TypeOf(windowsAttributes), &waValue, "windows") + return windowsAttributes, unknownAttribs, err +} + +// restoreCreationTime gets the creation time from the data and sets it to the file/folder at +// the specified path. +func restoreCreationTime(path string, creationTime *syscall.Filetime) (err error) { + pathPointer, err := syscall.UTF16PtrFromString(path) + if err != nil { + return err + } + handle, err := syscall.CreateFile(pathPointer, + syscall.FILE_WRITE_ATTRIBUTES, syscall.FILE_SHARE_WRITE, nil, + syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0) + if err != nil { + return err + } + defer func() { + if err := syscall.Close(handle); err != nil { + debug.Log("Error closing file handle for %s: %v\n", path, err) + } + }() + return syscall.SetFileTime(handle, creationTime, nil, nil) +} + +// restoreFileAttributes gets the File Attributes from the data and sets them to the file/folder +// at the specified path. +func restoreFileAttributes(path string, fileAttributes *uint32) (err error) { + pathPointer, err := syscall.UTF16PtrFromString(path) + if err != nil { + return err + } + err = fixEncryptionAttribute(path, fileAttributes, pathPointer) + if err != nil { + debug.Log("Could not change encryption attribute for path: %s: %v", path, err) + } + return syscall.SetFileAttributes(pathPointer, *fileAttributes) +} + +// fixEncryptionAttribute checks if a file needs to be marked encrypted and is not already encrypted, it sets +// the FILE_ATTRIBUTE_ENCRYPTED. Conversely, if the file needs to be marked unencrypted and it is already +// marked encrypted, it removes the FILE_ATTRIBUTE_ENCRYPTED. +func fixEncryptionAttribute(path string, attrs *uint32, pathPointer *uint16) (err error) { + if *attrs&windows.FILE_ATTRIBUTE_ENCRYPTED != 0 { + // File should be encrypted. + err = encryptFile(pathPointer) + if err != nil { + if fs.IsAccessDenied(err) { + // If existing file already has readonly or system flag, encrypt file call fails. + // We have already cleared readonly flag, clearing system flag if needed. + // The readonly and system flags will be set again at the end of this func if they are needed. + err = fs.ClearSystem(path) + if err != nil { + return fmt.Errorf("failed to encrypt file: failed to clear system flag: %s : %v", path, err) + } + err = encryptFile(pathPointer) + if err != nil { + return fmt.Errorf("failed to encrypt file: %s : %v", path, err) + } + } else { + return fmt.Errorf("failed to encrypt file: %s : %v", path, err) + } + } + } else { + existingAttrs, err := windows.GetFileAttributes(pathPointer) + if err != nil { + return fmt.Errorf("failed to get file attributes for existing file: %s : %v", path, err) + } + if existingAttrs&windows.FILE_ATTRIBUTE_ENCRYPTED != 0 { + // File should not be encrypted, but its already encrypted. Decrypt it. + err = decryptFile(pathPointer) + if err != nil { + if fs.IsAccessDenied(err) { + // If existing file already has readonly or system flag, decrypt file call fails. + // We have already cleared readonly flag, clearing system flag if needed. + // The readonly and system flags will be set again after this func if they are needed. + err = fs.ClearSystem(path) + if err != nil { + return fmt.Errorf("failed to decrypt file: failed to clear system flag: %s : %v", path, err) + } + err = decryptFile(pathPointer) + if err != nil { + return fmt.Errorf("failed to decrypt file: %s : %v", path, err) + } + } else { + return fmt.Errorf("failed to decrypt file: %s : %v", path, err) + } + } + } + } + return err +} + +// encryptFile set the encrypted flag on the file. +func encryptFile(pathPointer *uint16) error { + // Call EncryptFile function + ret, _, err := procEncryptFile.Call(uintptr(unsafe.Pointer(pathPointer))) + if ret == 0 { + return err + } + return nil +} + +// decryptFile removes the encrypted flag from the file. +func decryptFile(pathPointer *uint16) error { + // Call DecryptFile function + ret, _, err := procDecryptFile.Call(uintptr(unsafe.Pointer(pathPointer))) + if ret == 0 { + return err + } + return nil +} + +// fillGenericAttributes fills in the generic attributes for windows like File Attributes, +// Created time etc. +func (node *Node) fillGenericAttributes(path string, fi os.FileInfo, stat *statT) (allowExtended bool, err error) { + if strings.Contains(filepath.Base(path), ":") { + //Do not process for Alternate Data Streams in Windows + // Also do not allow processing of extended attributes for ADS. + return false, nil + } + if !strings.HasSuffix(filepath.Clean(path), `\`) { + // Do not process file attributes and created time for windows directories like + // C:, D: + // Filepath.Clean(path) ends with '\' for Windows root drives only. + + // Add Windows attributes + node.GenericAttributes, err = WindowsAttrsToGenericAttributes(WindowsAttributes{ + CreationTime: getCreationTime(fi, path), + FileAttributes: &stat.FileAttributes, + }) + } + return true, err +} + +// windowsAttrsToGenericAttributes converts the WindowsAttributes to a generic attributes map using reflection +func WindowsAttrsToGenericAttributes(windowsAttributes WindowsAttributes) (attrs map[GenericAttributeType]json.RawMessage, err error) { + // Get the value of the WindowsAttributes + windowsAttributesValue := reflect.ValueOf(windowsAttributes) + return osAttrsToGenericAttributes(reflect.TypeOf(windowsAttributes), &windowsAttributesValue, runtime.GOOS) +} + +// getCreationTime gets the value for the WindowsAttribute CreationTime in a windows specific time format. +// The value is a 64-bit value representing the number of 100-nanosecond intervals since January 1, 1601 (UTC) +// split into two 32-bit parts: the low-order DWORD and the high-order DWORD for efficiency and interoperability. +// The low-order DWORD represents the number of 100-nanosecond intervals elapsed since January 1, 1601, modulo +// 2^32. The high-order DWORD represents the number of times the low-order DWORD has overflowed. +func getCreationTime(fi os.FileInfo, path string) (creationTimeAttribute *syscall.Filetime) { + attrib, success := fi.Sys().(*syscall.Win32FileAttributeData) + if success && attrib != nil { + return &attrib.CreationTime + } else { + debug.Log("Could not get create time for path: %s", path) + return nil + } +} diff --git a/internal/restic/node_windows_test.go b/internal/restic/node_windows_test.go new file mode 100644 index 000000000..501d5a98a --- /dev/null +++ b/internal/restic/node_windows_test.go @@ -0,0 +1,210 @@ +//go:build windows +// +build windows + +package restic + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "syscall" + "testing" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/test" + "golang.org/x/sys/windows" +) + +func TestRestoreCreationTime(t *testing.T) { + t.Parallel() + path := t.TempDir() + fi, err := os.Lstat(path) + test.OK(t, errors.Wrapf(err, "Could not Lstat for path: %s", path)) + creationTimeAttribute := getCreationTime(fi, path) + test.OK(t, errors.Wrapf(err, "Could not get creation time for path: %s", path)) + //Using the temp dir creation time as the test creation time for the test file and folder + runGenericAttributesTest(t, path, TypeCreationTime, WindowsAttributes{CreationTime: creationTimeAttribute}, false) +} + +func TestRestoreFileAttributes(t *testing.T) { + t.Parallel() + genericAttributeName := TypeFileAttributes + tempDir := t.TempDir() + normal := uint32(syscall.FILE_ATTRIBUTE_NORMAL) + hidden := uint32(syscall.FILE_ATTRIBUTE_HIDDEN) + system := uint32(syscall.FILE_ATTRIBUTE_SYSTEM) + archive := uint32(syscall.FILE_ATTRIBUTE_ARCHIVE) + encrypted := uint32(windows.FILE_ATTRIBUTE_ENCRYPTED) + fileAttributes := []WindowsAttributes{ + //normal + {FileAttributes: &normal}, + //hidden + {FileAttributes: &hidden}, + //system + {FileAttributes: &system}, + //archive + {FileAttributes: &archive}, + //encrypted + {FileAttributes: &encrypted}, + } + for i, fileAttr := range fileAttributes { + genericAttrs, err := WindowsAttrsToGenericAttributes(fileAttr) + test.OK(t, err) + expectedNodes := []Node{ + { + Name: fmt.Sprintf("testfile%d", i), + Type: "file", + Mode: 0655, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: genericAttrs, + }, + } + runGenericAttributesTestForNodes(t, expectedNodes, tempDir, genericAttributeName, fileAttr, false) + } + normal = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY) + hidden = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | syscall.FILE_ATTRIBUTE_HIDDEN) + system = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | windows.FILE_ATTRIBUTE_SYSTEM) + archive = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | windows.FILE_ATTRIBUTE_ARCHIVE) + encrypted = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | windows.FILE_ATTRIBUTE_ENCRYPTED) + folderAttributes := []WindowsAttributes{ + //normal + {FileAttributes: &normal}, + //hidden + {FileAttributes: &hidden}, + //system + {FileAttributes: &system}, + //archive + {FileAttributes: &archive}, + //encrypted + {FileAttributes: &encrypted}, + } + for i, folderAttr := range folderAttributes { + genericAttrs, err := WindowsAttrsToGenericAttributes(folderAttr) + test.OK(t, err) + expectedNodes := []Node{ + { + Name: fmt.Sprintf("testdirectory%d", i), + Type: "dir", + Mode: 0755, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: genericAttrs, + }, + } + runGenericAttributesTestForNodes(t, expectedNodes, tempDir, genericAttributeName, folderAttr, false) + } +} + +func runGenericAttributesTest(t *testing.T, tempDir string, genericAttributeName GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { + genericAttributes, err := WindowsAttrsToGenericAttributes(genericAttributeExpected) + test.OK(t, err) + expectedNodes := []Node{ + { + Name: "testfile", + Type: "file", + Mode: 0644, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: genericAttributes, + }, + { + Name: "testdirectory", + Type: "dir", + Mode: 0755, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: genericAttributes, + }, + } + runGenericAttributesTestForNodes(t, expectedNodes, tempDir, genericAttributeName, genericAttributeExpected, warningExpected) +} +func runGenericAttributesTestForNodes(t *testing.T, expectedNodes []Node, tempDir string, genericAttr GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { + + for _, testNode := range expectedNodes { + testPath, node := restoreAndGetNode(t, tempDir, testNode, warningExpected) + rawMessage := node.GenericAttributes[genericAttr] + genericAttrsExpected, err := WindowsAttrsToGenericAttributes(genericAttributeExpected) + test.OK(t, err) + rawMessageExpected := genericAttrsExpected[genericAttr] + test.Equals(t, rawMessageExpected, rawMessage, "Generic attribute: %s got from NodeFromFileInfo not equal for path: %s", string(genericAttr), testPath) + } +} + +func restoreAndGetNode(t *testing.T, tempDir string, testNode Node, warningExpected bool) (string, *Node) { + testPath := filepath.Join(tempDir, "001", testNode.Name) + err := os.MkdirAll(filepath.Dir(testPath), testNode.Mode) + test.OK(t, errors.Wrapf(err, "Failed to create parent directories for: %s", testPath)) + + if testNode.Type == "file" { + + testFile, err := os.Create(testPath) + test.OK(t, errors.Wrapf(err, "Failed to create test file: %s", testPath)) + testFile.Close() + } else if testNode.Type == "dir" { + + err := os.Mkdir(testPath, testNode.Mode) + test.OK(t, errors.Wrapf(err, "Failed to create test directory: %s", testPath)) + } + + err = testNode.RestoreMetadata(testPath, func(msg string) { + if warningExpected { + test.Assert(t, warningExpected, "Warning triggered as expected: %s", msg) + } else { + // If warning is not expected, this code should not get triggered. + test.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", testPath, msg)) + } + }) + test.OK(t, errors.Wrapf(err, "Failed to restore metadata for: %s", testPath)) + + fi, err := os.Lstat(testPath) + test.OK(t, errors.Wrapf(err, "Could not Lstat for path: %s", testPath)) + + nodeFromFileInfo, err := NodeFromFileInfo(testPath, fi) + test.OK(t, errors.Wrapf(err, "Could not get NodeFromFileInfo for path: %s", testPath)) + + return testPath, nodeFromFileInfo +} + +const TypeSomeNewAttribute GenericAttributeType = "MockAttributes.SomeNewAttribute" + +func TestNewGenericAttributeType(t *testing.T) { + t.Parallel() + + newGenericAttribute := map[GenericAttributeType]json.RawMessage{} + newGenericAttribute[TypeSomeNewAttribute] = []byte("any value") + + tempDir := t.TempDir() + expectedNodes := []Node{ + { + Name: "testfile", + Type: "file", + Mode: 0644, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: newGenericAttribute, + }, + { + Name: "testdirectory", + Type: "dir", + Mode: 0755, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: newGenericAttribute, + }, + } + for _, testNode := range expectedNodes { + testPath, node := restoreAndGetNode(t, tempDir, testNode, true) + _, ua, err := genericAttributesToWindowsAttrs(node.GenericAttributes) + test.OK(t, err) + // Since this GenericAttribute is unknown to this version of the software, it will not get set on the file. + test.Assert(t, len(ua) == 0, "Unkown attributes: %s found for path: %s", ua, testPath) + } +} diff --git a/internal/restic/node_xattr.go b/internal/restic/node_xattr.go index ea9eafe94..0b2d5d552 100644 --- a/internal/restic/node_xattr.go +++ b/internal/restic/node_xattr.go @@ -4,6 +4,7 @@ package restic import ( + "os" "syscall" "github.com/restic/restic/internal/errors" @@ -47,3 +48,13 @@ func handleXattrErr(err error) error { return errors.WithStack(e) } } + +// restoreGenericAttributes is no-op. +func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { + return node.handleAllUnknownGenericAttributesFound(warn) +} + +// fillGenericAttributes is a no-op. +func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { + return true, nil +} diff --git a/internal/restic/parallel.go b/internal/restic/parallel.go index b22a249fe..cefbf0358 100644 --- a/internal/restic/parallel.go +++ b/internal/restic/parallel.go @@ -3,7 +3,9 @@ package restic import ( "context" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/ui/progress" "golang.org/x/sync/errgroup" ) @@ -50,3 +52,43 @@ func ParallelList(ctx context.Context, r Lister, t FileType, parallelism uint, f return wg.Wait() } + +// ParallelRemove deletes the given fileList of fileType in parallel +// if callback returns an error, then it will abort. +func ParallelRemove(ctx context.Context, repo Repository, fileList IDSet, fileType FileType, report func(id ID, err error) error, bar *progress.Counter) error { + fileChan := make(chan ID) + wg, ctx := errgroup.WithContext(ctx) + wg.Go(func() error { + defer close(fileChan) + for id := range fileList { + select { + case fileChan <- id: + case <-ctx.Done(): + return ctx.Err() + } + } + return nil + }) + + bar.SetMax(uint64(len(fileList))) + + // deleting files is IO-bound + workerCount := repo.Connections() + for i := 0; i < int(workerCount); i++ { + wg.Go(func() error { + for id := range fileChan { + h := backend.Handle{Type: fileType, Name: id.String()} + err := repo.Backend().Remove(ctx, h) + if report != nil { + err = report(id, err) + } + if err != nil { + return err + } + bar.Add(1) + } + return nil + }) + } + return wg.Wait() +} diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 895c930dd..66cc22ea9 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -44,6 +44,7 @@ type Repository interface { ListPack(context.Context, ID, int64) ([]Blob, uint32, error) LoadBlob(context.Context, BlobType, ID, []byte) ([]byte, error) + LoadBlobsFromPack(ctx context.Context, packID ID, blobs []Blob, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error SaveBlob(context.Context, BlobType, []byte, ID, bool) (ID, bool, int, error) // StartPackUploader start goroutines to upload new pack files. The errgroup @@ -88,6 +89,13 @@ type PackBlobs struct { Blobs []Blob } +type MasterIndexSaveOpts struct { + SaveProgress *progress.Counter + DeleteProgress func() *progress.Counter + DeleteReport func(id ID, err error) + SkipDeletion bool +} + // MasterIndex keeps track of the blobs are stored within files. type MasterIndex interface { Has(BlobHandle) bool @@ -98,10 +106,15 @@ type MasterIndex interface { Each(ctx context.Context, fn func(PackedBlob)) ListPacks(ctx context.Context, packs IDSet) <-chan PackBlobs - Save(ctx context.Context, repo SaverUnpacked, packBlacklist IDSet, extraObsolete IDs, p *progress.Counter) (obsolete IDSet, err error) + Save(ctx context.Context, repo Repository, excludePacks IDSet, extraObsolete IDs, opts MasterIndexSaveOpts) error } // Lister allows listing files in a backend. type Lister interface { List(ctx context.Context, t FileType, fn func(ID, int64) error) error } + +type ListerLoaderUnpacked interface { + Lister + LoaderUnpacked +} diff --git a/internal/restic/snapshot.go b/internal/restic/snapshot.go index 88171a646..8cf651d96 100644 --- a/internal/restic/snapshot.go +++ b/internal/restic/snapshot.go @@ -83,7 +83,7 @@ func ForAllSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked, excl var m sync.Mutex // For most snapshots decoding is nearly for free, thus just assume were only limited by IO - return ParallelList(ctx, be, SnapshotFile, loader.Connections(), func(ctx context.Context, id ID, size int64) error { + return ParallelList(ctx, be, SnapshotFile, loader.Connections(), func(ctx context.Context, id ID, _ int64) error { if excludeIDs.Has(id) { return nil } diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index 99a460321..f2c134ea9 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -7,7 +7,6 @@ import ( "golang.org/x/sync/errgroup" - "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" @@ -45,11 +44,12 @@ type packInfo struct { files map[*fileInfo]struct{} // set of files that use blobs from this pack } +type blobsLoaderFn func(ctx context.Context, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error + // fileRestorer restores set of files type fileRestorer struct { - key *crypto.Key - idx func(restic.BlobHandle) []restic.PackedBlob - packLoader repository.BackendLoadFn + idx func(restic.BlobHandle) []restic.PackedBlob + blobsLoader blobsLoaderFn workerCount int filesWriter *filesWriter @@ -63,8 +63,7 @@ type fileRestorer struct { } func newFileRestorer(dst string, - packLoader repository.BackendLoadFn, - key *crypto.Key, + blobsLoader blobsLoaderFn, idx func(restic.BlobHandle) []restic.PackedBlob, connections uint, sparse bool, @@ -74,9 +73,8 @@ func newFileRestorer(dst string, workerCount := int(connections) return &fileRestorer{ - key: key, idx: idx, - packLoader: packLoader, + blobsLoader: blobsLoader, filesWriter: newFilesWriter(workerCount), zeroChunk: repository.ZeroChunk(), sparse: sparse, @@ -310,7 +308,7 @@ func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID, for _, entry := range blobs { blobList = append(blobList, entry.blob) } - return repository.StreamPack(ctx, r.packLoader, r.key, packID, blobList, + return r.blobsLoader(ctx, packID, blobList, func(h restic.BlobHandle, blobData []byte, err error) error { processedBlobs.Insert(h) blob := blobs[h.ID] diff --git a/internal/restorer/filerestorer_test.go b/internal/restorer/filerestorer_test.go index c5bc3fe31..befeb5d2c 100644 --- a/internal/restorer/filerestorer_test.go +++ b/internal/restorer/filerestorer_test.go @@ -4,14 +4,11 @@ import ( "bytes" "context" "fmt" - "io" "os" + "sort" "testing" - "github.com/restic/restic/internal/backend" - "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -27,11 +24,6 @@ type TestFile struct { } type TestRepo struct { - key *crypto.Key - - // pack names and ids - packsNameToID map[string]restic.ID - packsIDToName map[restic.ID]string packsIDToData map[restic.ID][]byte // blobs and files @@ -40,7 +32,7 @@ type TestRepo struct { filesPathToContent map[string]string // - loader repository.BackendLoadFn + loader blobsLoaderFn } func (i *TestRepo) Lookup(bh restic.BlobHandle) []restic.PackedBlob { @@ -59,16 +51,6 @@ func newTestRepo(content []TestFile) *TestRepo { blobs map[restic.ID]restic.Blob } packs := make(map[string]Pack) - - key := crypto.NewRandomKey() - seal := func(data []byte) []byte { - ciphertext := crypto.NewBlobBuffer(len(data)) - ciphertext = ciphertext[:0] // truncate the slice - nonce := crypto.NewRandomNonce() - ciphertext = append(ciphertext, nonce...) - return key.Seal(ciphertext, nonce, data, nil) - } - filesPathToContent := make(map[string]string) for _, file := range content { @@ -86,14 +68,15 @@ func newTestRepo(content []TestFile) *TestRepo { // calculate blob id and add to the pack as necessary blobID := restic.Hash([]byte(blob.data)) if _, found := pack.blobs[blobID]; !found { - blobData := seal([]byte(blob.data)) + blobData := []byte(blob.data) pack.blobs[blobID] = restic.Blob{ BlobHandle: restic.BlobHandle{ Type: restic.DataBlob, ID: blobID, }, - Length: uint(len(blobData)), - Offset: uint(len(pack.data)), + Length: uint(len(blobData)), + UncompressedLength: uint(len(blobData)), + Offset: uint(len(pack.data)), } pack.data = append(pack.data, blobData...) } @@ -104,15 +87,11 @@ func newTestRepo(content []TestFile) *TestRepo { } blobs := make(map[restic.ID][]restic.PackedBlob) - packsIDToName := make(map[restic.ID]string) packsIDToData := make(map[restic.ID][]byte) - packsNameToID := make(map[string]restic.ID) for _, pack := range packs { packID := restic.Hash(pack.data) - packsIDToName[packID] = pack.name packsIDToData[packID] = pack.data - packsNameToID[pack.name] = packID for blobID, blob := range pack.blobs { blobs[blobID] = append(blobs[blobID], restic.PackedBlob{Blob: blob, PackID: packID}) } @@ -128,30 +107,44 @@ func newTestRepo(content []TestFile) *TestRepo { } repo := &TestRepo{ - key: key, - packsIDToName: packsIDToName, packsIDToData: packsIDToData, - packsNameToID: packsNameToID, blobs: blobs, files: files, filesPathToContent: filesPathToContent, } - repo.loader = func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - packID, err := restic.ParseID(h.Name) - if err != nil { - return err + repo.loader = func(ctx context.Context, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { + blobs = append([]restic.Blob{}, blobs...) + sort.Slice(blobs, func(i, j int) bool { + return blobs[i].Offset < blobs[j].Offset + }) + + for _, blob := range blobs { + found := false + for _, e := range repo.blobs[blob.ID] { + if packID == e.PackID { + found = true + buf := repo.packsIDToData[packID][e.Offset : e.Offset+e.Length] + err := handleBlobFn(e.BlobHandle, buf, nil) + if err != nil { + return err + } + } + } + if !found { + return fmt.Errorf("missing blob: %v", blob) + } } - rd := bytes.NewReader(repo.packsIDToData[packID][int(offset) : int(offset)+length]) - return fn(rd) + return nil } return repo } func restoreAndVerify(t *testing.T, tempdir string, content []TestFile, files map[string]bool, sparse bool) { + t.Helper() repo := newTestRepo(content) - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, sparse, nil) + r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, sparse, nil) if files == nil { r.files = repo.files @@ -170,6 +163,7 @@ func restoreAndVerify(t *testing.T, tempdir string, content []TestFile, files ma } func verifyRestore(t *testing.T, r *fileRestorer, repo *TestRepo) { + t.Helper() for _, file := range r.files { target := r.targetPath(file.location) data, err := os.ReadFile(target) @@ -283,62 +277,17 @@ func TestErrorRestoreFiles(t *testing.T) { loadError := errors.New("load error") // loader always returns an error - repo.loader = func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + repo.loader = func(ctx context.Context, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { return loadError } - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false, nil) + r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, false, nil) r.files = repo.files err := r.restoreFiles(context.TODO()) rtest.Assert(t, errors.Is(err, loadError), "got %v, expected contained error %v", err, loadError) } -func TestDownloadError(t *testing.T) { - for i := 0; i < 100; i += 10 { - testPartialDownloadError(t, i) - } -} - -func testPartialDownloadError(t *testing.T, part int) { - tempdir := rtest.TempDir(t) - content := []TestFile{ - { - name: "file1", - blobs: []TestBlob{ - {"data1-1", "pack1"}, - {"data1-2", "pack1"}, - {"data1-3", "pack1"}, - }, - }} - - repo := newTestRepo(content) - - // loader always returns an error - loader := repo.loader - repo.loader = func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - // only load partial data to exercise fault handling in different places - err := loader(ctx, h, length*part/100, offset, fn) - if err == nil { - return nil - } - fmt.Println("Retry after error", err) - return loader(ctx, h, length, offset, fn) - } - - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false, nil) - r.files = repo.files - r.Error = func(s string, e error) error { - // ignore errors as in the `restore` command - fmt.Println("error during restore", s, e) - return nil - } - - err := r.restoreFiles(context.TODO()) - rtest.OK(t, err) - verifyRestore(t, r, repo) -} - func TestFatalDownloadError(t *testing.T) { tempdir := rtest.TempDir(t) content := []TestFile{ @@ -361,12 +310,19 @@ func TestFatalDownloadError(t *testing.T) { repo := newTestRepo(content) loader := repo.loader - repo.loader = func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - // only return half the data to break file2 - return loader(ctx, h, length/2, offset, fn) + repo.loader = func(ctx context.Context, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { + ctr := 0 + return loader(ctx, packID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { + if ctr < 2 { + ctr++ + return handleBlobFn(blob, buf, err) + } + // break file2 + return errors.New("failed to load blob") + }) } - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false, nil) + r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, false, nil) r.files = repo.files var errors []string diff --git a/internal/restorer/fileswriter.go b/internal/restorer/fileswriter.go index 589aa502a..cbe89c30c 100644 --- a/internal/restorer/fileswriter.go +++ b/internal/restorer/fileswriter.go @@ -50,16 +50,26 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create bucket.files[path].users++ return wr, nil } - - var flags int + var f *os.File + var err error if createSize >= 0 { - flags = os.O_CREATE | os.O_TRUNC | os.O_WRONLY - } else { - flags = os.O_WRONLY - } - - f, err := os.OpenFile(path, flags, 0600) - if err != nil { + if f, err = os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600); err != nil { + if fs.IsAccessDenied(err) { + // If file is readonly, clear the readonly flag by resetting the + // permissions of the file and try again + // as the metadata will be set again in the second pass and the + // readonly flag will be applied again if needed. + if err = fs.ResetPermissions(path); err != nil { + return nil, err + } + if f, err = os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0600); err != nil { + return nil, err + } + } else { + return nil, err + } + } + } else if f, err = os.OpenFile(path, os.O_WRONLY, 0600); err != nil { return nil, err } diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index e973316c0..9f41f5cf2 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -24,10 +24,11 @@ type Restorer struct { progress *restoreui.Progress Error func(location string, err error) error + Warn func(message string) SelectFilter func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) } -var restorerAbortOnAllErrors = func(location string, err error) error { return err } +var restorerAbortOnAllErrors = func(_ string, err error) error { return err } // NewRestorer creates a restorer preloaded with the content from the snapshot id. func NewRestorer(repo restic.Repository, sn *restic.Snapshot, sparse bool, @@ -178,7 +179,7 @@ func (res *Restorer) restoreNodeTo(ctx context.Context, node *restic.Node, targe func (res *Restorer) restoreNodeMetadataTo(node *restic.Node, target, location string) error { debug.Log("restoreNodeMetadata %v %v %v", node.Name, target, location) - err := node.RestoreMetadata(target) + err := node.RestoreMetadata(target, res.Warn) if err != nil { debug.Log("node.RestoreMetadata(%s) error %v", target, err) } @@ -204,11 +205,19 @@ func (res *Restorer) restoreHardlinkAt(node *restic.Node, target, path, location func (res *Restorer) restoreEmptyFileAt(node *restic.Node, target, location string) error { wr, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) - if err != nil { - return err + if fs.IsAccessDenied(err) { + // If file is readonly, clear the readonly flag by resetting the + // permissions of the file and try again + // as the metadata will be set again in the second pass and the + // readonly flag will be applied again if needed. + if err = fs.ResetPermissions(target); err != nil { + return err + } + if wr, err = os.OpenFile(target, os.O_TRUNC|os.O_WRONLY, 0600); err != nil { + return err + } } - err = wr.Close() - if err != nil { + if err = wr.Close(); err != nil { return err } @@ -231,7 +240,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { } idx := NewHardlinkIndex[string]() - filerestorer := newFileRestorer(dst, res.repo.Backend().Load, res.repo.Key(), res.repo.Index().Lookup, + filerestorer := newFileRestorer(dst, res.repo.LoadBlobsFromPack, res.repo.Index().Lookup, res.repo.Connections(), res.sparse, res.progress) filerestorer.Error = res.Error @@ -239,7 +248,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { // first tree pass: create directories and collect all files to restore _, err = res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{ - enterDir: func(node *restic.Node, target, location string) error { + enterDir: func(_ *restic.Node, target, location string) error { debug.Log("first pass, enterDir: mkdir %q, leaveDir should restore metadata", location) if res.progress != nil { res.progress.AddFile(0) @@ -366,7 +375,7 @@ func (res *Restorer) VerifyFiles(ctx context.Context, dst string) (int, error) { defer close(work) _, err := res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{ - visitNode: func(node *restic.Node, target, location string) error { + visitNode: func(node *restic.Node, target, _ string) error { if node.Type != "file" { return nil } diff --git a/internal/restorer/restorer_test.go b/internal/restorer/restorer_test.go index d0e7dad6f..5742d7663 100644 --- a/internal/restorer/restorer_test.go +++ b/internal/restorer/restorer_test.go @@ -3,6 +3,7 @@ package restorer import ( "bytes" "context" + "encoding/json" "io" "math" "os" @@ -27,20 +28,30 @@ type Snapshot struct { } type File struct { - Data string - Links uint64 - Inode uint64 - Mode os.FileMode - ModTime time.Time + Data string + Links uint64 + Inode uint64 + Mode os.FileMode + ModTime time.Time + attributes *FileAttributes } type Dir struct { - Nodes map[string]Node - Mode os.FileMode - ModTime time.Time + Nodes map[string]Node + Mode os.FileMode + ModTime time.Time + attributes *FileAttributes } -func saveFile(t testing.TB, repo restic.Repository, node File) restic.ID { +type FileAttributes struct { + ReadOnly bool + Hidden bool + System bool + Archive bool + Encrypted bool +} + +func saveFile(t testing.TB, repo restic.BlobSaver, node File) restic.ID { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -52,7 +63,7 @@ func saveFile(t testing.TB, repo restic.Repository, node File) restic.ID { return id } -func saveDir(t testing.TB, repo restic.Repository, nodes map[string]Node, inode uint64) restic.ID { +func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode uint64, getGenericAttributes func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage)) restic.ID { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -78,20 +89,21 @@ func saveDir(t testing.TB, repo restic.Repository, nodes map[string]Node, inode mode = 0644 } err := tree.Insert(&restic.Node{ - Type: "file", - Mode: mode, - ModTime: node.ModTime, - Name: name, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Content: fc, - Size: uint64(len(n.(File).Data)), - Inode: fi, - Links: lc, + Type: "file", + Mode: mode, + ModTime: node.ModTime, + Name: name, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Content: fc, + Size: uint64(len(n.(File).Data)), + Inode: fi, + Links: lc, + GenericAttributes: getGenericAttributes(node.attributes, false), }) rtest.OK(t, err) case Dir: - id := saveDir(t, repo, node.Nodes, inode) + id := saveDir(t, repo, node.Nodes, inode, getGenericAttributes) mode := node.Mode if mode == 0 { @@ -99,13 +111,14 @@ func saveDir(t testing.TB, repo restic.Repository, nodes map[string]Node, inode } err := tree.Insert(&restic.Node{ - Type: "dir", - Mode: mode, - ModTime: node.ModTime, - Name: name, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Subtree: &id, + Type: "dir", + Mode: mode, + ModTime: node.ModTime, + Name: name, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Subtree: &id, + GenericAttributes: getGenericAttributes(node.attributes, false), }) rtest.OK(t, err) default: @@ -121,13 +134,13 @@ func saveDir(t testing.TB, repo restic.Repository, nodes map[string]Node, inode return id } -func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot) (*restic.Snapshot, restic.ID) { +func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot, getGenericAttributes func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage)) (*restic.Snapshot, restic.ID) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() wg, wgCtx := errgroup.WithContext(ctx) repo.StartPackUploader(wgCtx, wg) - treeID := saveDir(t, repo, snapshot.Nodes, 1000) + treeID := saveDir(t, repo, snapshot.Nodes, 1000, getGenericAttributes) err := repo.Flush(ctx) if err != nil { t.Fatal(err) @@ -147,6 +160,11 @@ func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot) (*res return sn, id } +var noopGetGenericAttributes = func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage) { + // No-op + return nil +} + func TestRestorer(t *testing.T) { var tests = []struct { Snapshot @@ -322,7 +340,7 @@ func TestRestorer(t *testing.T) { for _, test := range tests { t.Run("", func(t *testing.T) { repo := repository.TestRepository(t) - sn, id := saveSnapshot(t, repo, test.Snapshot) + sn, id := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes) t.Logf("snapshot saved as %v", id.Str()) res := NewRestorer(repo, sn, false, nil) @@ -439,7 +457,7 @@ func TestRestorerRelative(t *testing.T) { t.Run("", func(t *testing.T) { repo := repository.TestRepository(t) - sn, id := saveSnapshot(t, repo, test.Snapshot) + sn, id := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes) t.Logf("snapshot saved as %v", id.Str()) res := NewRestorer(repo, sn, false, nil) @@ -669,7 +687,7 @@ func TestRestorerTraverseTree(t *testing.T) { for _, test := range tests { t.Run("", func(t *testing.T) { repo := repository.TestRepository(t) - sn, _ := saveSnapshot(t, repo, test.Snapshot) + sn, _ := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes) res := NewRestorer(repo, sn, false, nil) @@ -745,7 +763,7 @@ func TestRestorerConsistentTimestampsAndPermissions(t *testing.T) { }, }, }, - }) + }, noopGetGenericAttributes) res := NewRestorer(repo, sn, false, nil) @@ -800,7 +818,7 @@ func TestVerifyCancel(t *testing.T) { } repo := repository.TestRepository(t) - sn, _ := saveSnapshot(t, repo, snapshot) + sn, _ := saveSnapshot(t, repo, snapshot, noopGetGenericAttributes) res := NewRestorer(repo, sn, false, nil) diff --git a/internal/restorer/restorer_unix_test.go b/internal/restorer/restorer_unix_test.go index 2c30a6b64..0cbfefa92 100644 --- a/internal/restorer/restorer_unix_test.go +++ b/internal/restorer/restorer_unix_test.go @@ -29,7 +29,7 @@ func TestRestorerRestoreEmptyHardlinkedFileds(t *testing.T) { }, }, }, - }) + }, noopGetGenericAttributes) res := NewRestorer(repo, sn, false, nil) @@ -95,7 +95,7 @@ func TestRestorerProgressBar(t *testing.T) { }, "file2": File{Links: 1, Inode: 2, Data: "example"}, }, - }) + }, noopGetGenericAttributes) mock := &printerMock{} progress := restoreui.NewProgress(mock, 0) diff --git a/internal/restorer/restorer_windows_test.go b/internal/restorer/restorer_windows_test.go index 3ec4b1f11..684d51ace 100644 --- a/internal/restorer/restorer_windows_test.go +++ b/internal/restorer/restorer_windows_test.go @@ -4,11 +4,20 @@ package restorer import ( + "context" + "encoding/json" "math" + "os" + "path" "syscall" "testing" + "time" "unsafe" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test" "golang.org/x/sys/windows" ) @@ -33,3 +42,500 @@ func getBlockCount(t *testing.T, filename string) int64 { return int64(math.Ceil(float64(result) / 512)) } + +type DataStreamInfo struct { + name string + data string +} + +type NodeInfo struct { + DataStreamInfo + parentDir string + attributes FileAttributes + Exists bool + IsDirectory bool +} + +func TestFileAttributeCombination(t *testing.T) { + testFileAttributeCombination(t, false) +} + +func TestEmptyFileAttributeCombination(t *testing.T) { + testFileAttributeCombination(t, true) +} + +func testFileAttributeCombination(t *testing.T, isEmpty bool) { + t.Parallel() + //Generate combination of 5 attributes. + attributeCombinations := generateCombinations(5, []bool{}) + + fileName := "TestFile.txt" + // Iterate through each attribute combination + for _, attr1 := range attributeCombinations { + + //Set up the required file information + fileInfo := NodeInfo{ + DataStreamInfo: getDataStreamInfo(isEmpty, fileName), + parentDir: "dir", + attributes: getFileAttributes(attr1), + Exists: false, + } + + //Get the current test name + testName := getCombinationTestName(fileInfo, fileName, fileInfo.attributes) + + //Run test + t.Run(testName, func(t *testing.T) { + mainFilePath := runAttributeTests(t, fileInfo, fileInfo.attributes) + + verifyFileRestores(isEmpty, mainFilePath, t, fileInfo) + }) + } +} + +func generateCombinations(n int, prefix []bool) [][]bool { + if n == 0 { + // Return a slice containing the current permutation + return [][]bool{append([]bool{}, prefix...)} + } + + // Generate combinations with True + prefixTrue := append(prefix, true) + permsTrue := generateCombinations(n-1, prefixTrue) + + // Generate combinations with False + prefixFalse := append(prefix, false) + permsFalse := generateCombinations(n-1, prefixFalse) + + // Combine combinations with True and False + return append(permsTrue, permsFalse...) +} + +func getDataStreamInfo(isEmpty bool, fileName string) DataStreamInfo { + var dataStreamInfo DataStreamInfo + if isEmpty { + dataStreamInfo = DataStreamInfo{ + name: fileName, + } + } else { + dataStreamInfo = DataStreamInfo{ + name: fileName, + data: "Main file data stream.", + } + } + return dataStreamInfo +} + +func getFileAttributes(values []bool) FileAttributes { + return FileAttributes{ + ReadOnly: values[0], + Hidden: values[1], + System: values[2], + Archive: values[3], + Encrypted: values[4], + } +} + +func getCombinationTestName(fi NodeInfo, fileName string, overwriteAttr FileAttributes) string { + if fi.attributes.ReadOnly { + fileName += "-ReadOnly" + } + if fi.attributes.Hidden { + fileName += "-Hidden" + } + if fi.attributes.System { + fileName += "-System" + } + if fi.attributes.Archive { + fileName += "-Archive" + } + if fi.attributes.Encrypted { + fileName += "-Encrypted" + } + if fi.Exists { + fileName += "-Overwrite" + if overwriteAttr.ReadOnly { + fileName += "-R" + } + if overwriteAttr.Hidden { + fileName += "-H" + } + if overwriteAttr.System { + fileName += "-S" + } + if overwriteAttr.Archive { + fileName += "-A" + } + if overwriteAttr.Encrypted { + fileName += "-E" + } + } + return fileName +} + +func runAttributeTests(t *testing.T, fileInfo NodeInfo, existingFileAttr FileAttributes) string { + testDir := t.TempDir() + res, _ := setupWithFileAttributes(t, fileInfo, testDir, existingFileAttr) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := res.RestoreTo(ctx, testDir) + rtest.OK(t, err) + + mainFilePath := path.Join(testDir, fileInfo.parentDir, fileInfo.name) + //Verify restore + verifyFileAttributes(t, mainFilePath, fileInfo.attributes) + return mainFilePath +} + +func setupWithFileAttributes(t *testing.T, nodeInfo NodeInfo, testDir string, existingFileAttr FileAttributes) (*Restorer, []int) { + t.Helper() + if nodeInfo.Exists { + if !nodeInfo.IsDirectory { + err := os.MkdirAll(path.Join(testDir, nodeInfo.parentDir), os.ModeDir) + rtest.OK(t, err) + filepath := path.Join(testDir, nodeInfo.parentDir, nodeInfo.name) + if existingFileAttr.Encrypted { + err := createEncryptedFileWriteData(filepath, nodeInfo) + rtest.OK(t, err) + } else { + // Write the data to the file + file, err := os.OpenFile(path.Clean(filepath), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + rtest.OK(t, err) + _, err = file.Write([]byte(nodeInfo.data)) + rtest.OK(t, err) + + err = file.Close() + rtest.OK(t, err) + } + } else { + err := os.MkdirAll(path.Join(testDir, nodeInfo.parentDir, nodeInfo.name), os.ModeDir) + rtest.OK(t, err) + } + + pathPointer, err := syscall.UTF16PtrFromString(path.Join(testDir, nodeInfo.parentDir, nodeInfo.name)) + rtest.OK(t, err) + syscall.SetFileAttributes(pathPointer, getAttributeValue(&existingFileAttr)) + } + + index := 0 + + order := []int{} + streams := []DataStreamInfo{} + if !nodeInfo.IsDirectory { + order = append(order, index) + index++ + streams = append(streams, nodeInfo.DataStreamInfo) + } + return setup(t, getNodes(nodeInfo.parentDir, nodeInfo.name, order, streams, nodeInfo.IsDirectory, &nodeInfo.attributes)), order +} + +func createEncryptedFileWriteData(filepath string, fileInfo NodeInfo) (err error) { + var ptr *uint16 + if ptr, err = windows.UTF16PtrFromString(filepath); err != nil { + return err + } + var handle windows.Handle + //Create the file with encrypted flag + if handle, err = windows.CreateFile(ptr, uint32(windows.GENERIC_READ|windows.GENERIC_WRITE), uint32(windows.FILE_SHARE_READ), nil, uint32(windows.CREATE_ALWAYS), windows.FILE_ATTRIBUTE_ENCRYPTED, 0); err != nil { + return err + } + //Write data to file + if _, err = windows.Write(handle, []byte(fileInfo.data)); err != nil { + return err + } + //Close handle + return windows.CloseHandle(handle) +} + +func setup(t *testing.T, nodesMap map[string]Node) *Restorer { + repo := repository.TestRepository(t) + getFileAttributes := func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage) { + if attr == nil { + return + } + + fileattr := getAttributeValue(attr) + + if isDir { + //If the node is a directory add FILE_ATTRIBUTE_DIRECTORY to attributes + fileattr |= windows.FILE_ATTRIBUTE_DIRECTORY + } + attrs, err := restic.WindowsAttrsToGenericAttributes(restic.WindowsAttributes{FileAttributes: &fileattr}) + test.OK(t, err) + return attrs + } + sn, _ := saveSnapshot(t, repo, Snapshot{ + Nodes: nodesMap, + }, getFileAttributes) + res := NewRestorer(repo, sn, false, nil) + return res +} + +func getAttributeValue(attr *FileAttributes) uint32 { + var fileattr uint32 + if attr.ReadOnly { + fileattr |= windows.FILE_ATTRIBUTE_READONLY + } + if attr.Hidden { + fileattr |= windows.FILE_ATTRIBUTE_HIDDEN + } + if attr.Encrypted { + fileattr |= windows.FILE_ATTRIBUTE_ENCRYPTED + } + if attr.Archive { + fileattr |= windows.FILE_ATTRIBUTE_ARCHIVE + } + if attr.System { + fileattr |= windows.FILE_ATTRIBUTE_SYSTEM + } + return fileattr +} + +func getNodes(dir string, mainNodeName string, order []int, streams []DataStreamInfo, isDirectory bool, attributes *FileAttributes) map[string]Node { + var mode os.FileMode + if isDirectory { + mode = os.FileMode(2147484159) + } else { + if attributes != nil && attributes.ReadOnly { + mode = os.FileMode(0o444) + } else { + mode = os.FileMode(0o666) + } + } + + getFileNodes := func() map[string]Node { + nodes := map[string]Node{} + if isDirectory { + //Add a directory node at the same level as the other streams + nodes[mainNodeName] = Dir{ + ModTime: time.Now(), + attributes: attributes, + Mode: mode, + } + } + + if len(streams) > 0 { + for _, index := range order { + stream := streams[index] + + var attr *FileAttributes = nil + if mainNodeName == stream.name { + attr = attributes + } else if attributes != nil && attributes.Encrypted { + //Set encrypted attribute + attr = &FileAttributes{Encrypted: true} + } + + nodes[stream.name] = File{ + ModTime: time.Now(), + Data: stream.data, + Mode: mode, + attributes: attr, + } + } + } + return nodes + } + + return map[string]Node{ + dir: Dir{ + Mode: normalizeFileMode(0750 | mode), + ModTime: time.Now(), + Nodes: getFileNodes(), + }, + } +} + +func verifyFileAttributes(t *testing.T, mainFilePath string, attr FileAttributes) { + ptr, err := windows.UTF16PtrFromString(mainFilePath) + rtest.OK(t, err) + //Get file attributes using syscall + fileAttributes, err := syscall.GetFileAttributes(ptr) + rtest.OK(t, err) + //Test positive and negative scenarios + if attr.ReadOnly { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_READONLY != 0, "Expected read only attibute.") + } else { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_READONLY == 0, "Unexpected read only attibute.") + } + if attr.Hidden { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_HIDDEN != 0, "Expected hidden attibute.") + } else { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_HIDDEN == 0, "Unexpected hidden attibute.") + } + if attr.System { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_SYSTEM != 0, "Expected system attibute.") + } else { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_SYSTEM == 0, "Unexpected system attibute.") + } + if attr.Archive { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_ARCHIVE != 0, "Expected archive attibute.") + } else { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_ARCHIVE == 0, "Unexpected archive attibute.") + } + if attr.Encrypted { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_ENCRYPTED != 0, "Expected encrypted attibute.") + } else { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_ENCRYPTED == 0, "Unexpected encrypted attibute.") + } +} + +func verifyFileRestores(isEmpty bool, mainFilePath string, t *testing.T, fileInfo NodeInfo) { + if isEmpty { + _, err1 := os.Stat(mainFilePath) + rtest.Assert(t, !errors.Is(err1, os.ErrNotExist), "The file "+fileInfo.name+" does not exist") + } else { + + verifyMainFileRestore(t, mainFilePath, fileInfo) + } +} + +func verifyMainFileRestore(t *testing.T, mainFilePath string, fileInfo NodeInfo) { + fi, err1 := os.Stat(mainFilePath) + rtest.Assert(t, !errors.Is(err1, os.ErrNotExist), "The file "+fileInfo.name+" does not exist") + + size := fi.Size() + rtest.Assert(t, size > 0, "The file "+fileInfo.name+" exists but is empty") + + content, err := os.ReadFile(mainFilePath) + rtest.OK(t, err) + rtest.Assert(t, string(content) == fileInfo.data, "The file "+fileInfo.name+" exists but the content is not overwritten") +} + +func TestDirAttributeCombination(t *testing.T) { + t.Parallel() + attributeCombinations := generateCombinations(4, []bool{}) + + dirName := "TestDir" + // Iterate through each attribute combination + for _, attr1 := range attributeCombinations { + + //Set up the required directory information + dirInfo := NodeInfo{ + DataStreamInfo: DataStreamInfo{ + name: dirName, + }, + parentDir: "dir", + attributes: getDirFileAttributes(attr1), + Exists: false, + IsDirectory: true, + } + + //Get the current test name + testName := getCombinationTestName(dirInfo, dirName, dirInfo.attributes) + + //Run test + t.Run(testName, func(t *testing.T) { + mainDirPath := runAttributeTests(t, dirInfo, dirInfo.attributes) + + //Check directory exists + _, err1 := os.Stat(mainDirPath) + rtest.Assert(t, !errors.Is(err1, os.ErrNotExist), "The directory "+dirInfo.name+" does not exist") + }) + } +} + +func getDirFileAttributes(values []bool) FileAttributes { + return FileAttributes{ + // readonly not valid for directories + Hidden: values[0], + System: values[1], + Archive: values[2], + Encrypted: values[3], + } +} + +func TestFileAttributeCombinationsOverwrite(t *testing.T) { + testFileAttributeCombinationsOverwrite(t, false) +} + +func TestEmptyFileAttributeCombinationsOverwrite(t *testing.T) { + testFileAttributeCombinationsOverwrite(t, true) +} + +func testFileAttributeCombinationsOverwrite(t *testing.T, isEmpty bool) { + t.Parallel() + //Get attribute combinations + attributeCombinations := generateCombinations(5, []bool{}) + //Get overwrite file attribute combinations + overwriteCombinations := generateCombinations(5, []bool{}) + + fileName := "TestOverwriteFile" + + //Iterate through each attribute combination + for _, attr1 := range attributeCombinations { + + fileInfo := NodeInfo{ + DataStreamInfo: getDataStreamInfo(isEmpty, fileName), + parentDir: "dir", + attributes: getFileAttributes(attr1), + Exists: true, + } + + overwriteFileAttributes := []FileAttributes{} + + for _, overwrite := range overwriteCombinations { + overwriteFileAttributes = append(overwriteFileAttributes, getFileAttributes(overwrite)) + } + + //Iterate through each overwrite attribute combination + for _, overwriteFileAttr := range overwriteFileAttributes { + //Get the test name + testName := getCombinationTestName(fileInfo, fileName, overwriteFileAttr) + + //Run test + t.Run(testName, func(t *testing.T) { + mainFilePath := runAttributeTests(t, fileInfo, overwriteFileAttr) + + verifyFileRestores(isEmpty, mainFilePath, t, fileInfo) + }) + } + } +} + +func TestDirAttributeCombinationsOverwrite(t *testing.T) { + t.Parallel() + //Get attribute combinations + attributeCombinations := generateCombinations(4, []bool{}) + //Get overwrite dir attribute combinations + overwriteCombinations := generateCombinations(4, []bool{}) + + dirName := "TestOverwriteDir" + + //Iterate through each attribute combination + for _, attr1 := range attributeCombinations { + + dirInfo := NodeInfo{ + DataStreamInfo: DataStreamInfo{ + name: dirName, + }, + parentDir: "dir", + attributes: getDirFileAttributes(attr1), + Exists: true, + IsDirectory: true, + } + + overwriteDirFileAttributes := []FileAttributes{} + + for _, overwrite := range overwriteCombinations { + overwriteDirFileAttributes = append(overwriteDirFileAttributes, getDirFileAttributes(overwrite)) + } + + //Iterate through each overwrite attribute combinations + for _, overwriteDirAttr := range overwriteDirFileAttributes { + //Get the test name + testName := getCombinationTestName(dirInfo, dirName, overwriteDirAttr) + + //Run test + t.Run(testName, func(t *testing.T) { + mainDirPath := runAttributeTests(t, dirInfo, dirInfo.attributes) + + //Check directory exists + _, err1 := os.Stat(mainDirPath) + rtest.Assert(t, !errors.Is(err1, os.ErrNotExist), "The directory "+dirInfo.name+" does not exist") + }) + } + } +} diff --git a/internal/test/helpers.go b/internal/test/helpers.go index 65e3e36ec..242da6079 100644 --- a/internal/test/helpers.go +++ b/internal/test/helpers.go @@ -3,6 +3,7 @@ package test import ( "compress/bzip2" "compress/gzip" + "fmt" "io" "os" "os/exec" @@ -47,10 +48,22 @@ func OKs(tb testing.TB, errs []error) { } // Equals fails the test if exp is not equal to act. -func Equals(tb testing.TB, exp, act interface{}) { +// msg is optional message to be printed, first param being format string and rest being arguments. +func Equals(tb testing.TB, exp, act interface{}, msgs ...string) { tb.Helper() if !reflect.DeepEqual(exp, act) { - tb.Fatalf("\033[31m\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", exp, act) + var msgString string + length := len(msgs) + if length == 1 { + msgString = msgs[0] + } else if length > 1 { + args := make([]interface{}, length-1) + for i, msg := range msgs[1:] { + args[i] = msg + } + msgString = fmt.Sprintf(msgs[0], args...) + } + tb.Fatalf("\033[31m\n\n\t"+msgString+"\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", exp, act) } } diff --git a/internal/ui/backup/progress.go b/internal/ui/backup/progress.go index 4362a8c83..da0d401a3 100644 --- a/internal/ui/backup/progress.go +++ b/internal/ui/backup/progress.go @@ -63,7 +63,7 @@ func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress { printer: printer, estimator: *newRateEstimator(time.Now()), } - p.Updater = *progress.NewUpdater(interval, func(runtime time.Duration, final bool) { + p.Updater = *progress.NewUpdater(interval, func(_ time.Duration, final bool) { if final { p.printer.Reset() } else { diff --git a/internal/ui/progress/printer.go b/internal/ui/progress/printer.go new file mode 100644 index 000000000..a671621e9 --- /dev/null +++ b/internal/ui/progress/printer.go @@ -0,0 +1,30 @@ +package progress + +// A Printer can can return a new counter or print messages +// at different log levels. +// It must be safe to call its methods from concurrent goroutines. +type Printer interface { + NewCounter(description string) *Counter + + E(msg string, args ...interface{}) + P(msg string, args ...interface{}) + V(msg string, args ...interface{}) + VV(msg string, args ...interface{}) +} + +// NoopPrinter discards all messages +type NoopPrinter struct{} + +var _ Printer = (*NoopPrinter)(nil) + +func (*NoopPrinter) NewCounter(_ string) *Counter { + return nil +} + +func (*NoopPrinter) E(_ string, _ ...interface{}) {} + +func (*NoopPrinter) P(_ string, _ ...interface{}) {} + +func (*NoopPrinter) V(_ string, _ ...interface{}) {} + +func (*NoopPrinter) VV(_ string, _ ...interface{}) {} diff --git a/internal/ui/termstatus/status.go b/internal/ui/termstatus/status.go index 95286de99..fc731b023 100644 --- a/internal/ui/termstatus/status.go +++ b/internal/ui/termstatus/status.go @@ -105,7 +105,7 @@ func (t *Terminal) run(ctx context.Context) { select { case <-ctx.Done(): if !IsProcessBackground(t.fd) { - t.undoStatus(len(status)) + t.writeStatus([]string{}) } return @@ -235,30 +235,6 @@ func (t *Terminal) runWithoutStatus(ctx context.Context) { } } -func (t *Terminal) undoStatus(lines int) { - for i := 0; i < lines; i++ { - t.clearCurrentLine(t.wr, t.fd) - - _, err := t.wr.WriteRune('\n') - if err != nil { - fmt.Fprintf(os.Stderr, "write failed: %v\n", err) - } - - // flush is needed so that the current line is updated - err = t.wr.Flush() - if err != nil { - fmt.Fprintf(os.Stderr, "flush failed: %v\n", err) - } - } - - t.moveCursorUp(t.wr, t.fd, lines) - - err := t.wr.Flush() - if err != nil { - fmt.Fprintf(os.Stderr, "flush failed: %v\n", err) - } -} - func (t *Terminal) print(line string, isErr bool) { // make sure the line ends with a line break if line[len(line)-1] != '\n' { diff --git a/internal/ui/termstatus/status_test.go b/internal/ui/termstatus/status_test.go index b59063076..997a2d7b1 100644 --- a/internal/ui/termstatus/status_test.go +++ b/internal/ui/termstatus/status_test.go @@ -39,11 +39,10 @@ func TestSetStatus(t *testing.T) { term.SetStatus([]string{"quux", "needs\nquote"}) exp += home + clear + "quux\n" + home + clear + "\"needs\\nquote\"\n" + - home + clear + home + up + up // Third line implicit. + home + clear + home + up + up // Clear third line cancel() - exp += home + clear + "\n" + home + clear + "\n" + - home + up + up // Status cleared. + exp += home + clear + "\n" + home + clear + home + up // Status cleared <-term.closed rtest.Equals(t, exp, buf.String()) diff --git a/internal/walker/rewriter.go b/internal/walker/rewriter.go index 649857032..6d283a625 100644 --- a/internal/walker/rewriter.go +++ b/internal/walker/rewriter.go @@ -39,13 +39,13 @@ func NewTreeRewriter(opts RewriteOpts) *TreeRewriter { } // setup default implementations if rw.opts.RewriteNode == nil { - rw.opts.RewriteNode = func(node *restic.Node, path string) *restic.Node { + rw.opts.RewriteNode = func(node *restic.Node, _ string) *restic.Node { return node } } if rw.opts.RewriteFailedTree == nil { // fail with error by default - rw.opts.RewriteFailedTree = func(nodeID restic.ID, path string, err error) (restic.ID, error) { + rw.opts.RewriteFailedTree = func(_ restic.ID, _ string, err error) (restic.ID, error) { return restic.ID{}, err } } diff --git a/internal/walker/walker.go b/internal/walker/walker.go index 4c4e7f5ab..091b05489 100644 --- a/internal/walker/walker.go +++ b/internal/walker/walker.go @@ -21,21 +21,22 @@ var ErrSkipNode = errors.New("skip this node") // When the special value ErrSkipNode is returned and node is a dir node, it is // not walked. When the node is not a dir node, the remaining items in this // tree are skipped. -// -// Setting ignore to true tells Walk that it should not visit the node again. -// For tree nodes, this means that the function is not called for the -// referenced tree. If the node is not a tree, and all nodes in the current -// tree have ignore set to true, the current tree will not be visited again. -// When err is not nil and different from ErrSkipNode, the value returned for -// ignore is ignored. -type WalkFunc func(parentTreeID restic.ID, path string, node *restic.Node, nodeErr error) (ignore bool, err error) +type WalkFunc func(parentTreeID restic.ID, path string, node *restic.Node, nodeErr error) (err error) + +type WalkVisitor struct { + // If the node is a `dir`, it will be entered afterwards unless `ErrSkipNode` + // was returned. This function is mandatory + ProcessNode WalkFunc + // Optional callback + LeaveDir func(path string) +} // Walk calls walkFn recursively for each node in root. If walkFn returns an // error, it is passed up the call stack. The trees in ignoreTrees are not // walked. If walkFn ignores trees, these are added to the set. -func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, ignoreTrees restic.IDSet, walkFn WalkFunc) error { +func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, visitor WalkVisitor) error { tree, err := restic.LoadTree(ctx, repo, root) - _, err = walkFn(root, "/", nil, err) + err = visitor.ProcessNode(root, "/", nil, err) if err != nil { if err == ErrSkipNode { @@ -44,24 +45,13 @@ func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, ignoreTre return err } - if ignoreTrees == nil { - ignoreTrees = restic.NewIDSet() - } - - _, err = walk(ctx, repo, "/", root, tree, ignoreTrees, walkFn) - return err + return walk(ctx, repo, "/", root, tree, visitor) } // walk recursively traverses the tree, ignoring subtrees when the ID of the // subtree is in ignoreTrees. If err is nil and ignore is true, the subtree ID // will be added to ignoreTrees by walk. -func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *restic.Tree, ignoreTrees restic.IDSet, walkFn WalkFunc) (ignore bool, err error) { - var allNodesIgnored = true - - if len(tree.Nodes) == 0 { - allNodesIgnored = false - } - +func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *restic.Tree, visitor WalkVisitor) (err error) { sort.Slice(tree.Nodes, func(i, j int) bool { return tree.Nodes[i].Name < tree.Nodes[j].Name }) @@ -70,68 +60,44 @@ func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTree p := path.Join(prefix, node.Name) if node.Type == "" { - return false, errors.Errorf("node type is empty for node %q", node.Name) + return errors.Errorf("node type is empty for node %q", node.Name) } if node.Type != "dir" { - ignore, err := walkFn(parentTreeID, p, node, nil) + err := visitor.ProcessNode(parentTreeID, p, node, nil) if err != nil { if err == ErrSkipNode { // skip the remaining entries in this tree - return allNodesIgnored, nil + break } - return false, err - } - - if !ignore { - allNodesIgnored = false + return err } continue } if node.Subtree == nil { - return false, errors.Errorf("subtree for node %v in tree %v is nil", node.Name, p) - } - - if ignoreTrees.Has(*node.Subtree) { - continue + return errors.Errorf("subtree for node %v in tree %v is nil", node.Name, p) } subtree, err := restic.LoadTree(ctx, repo, *node.Subtree) - ignore, err := walkFn(parentTreeID, p, node, err) + err = visitor.ProcessNode(parentTreeID, p, node, err) if err != nil { if err == ErrSkipNode { - if ignore { - ignoreTrees.Insert(*node.Subtree) - } continue } - return false, err } - if ignore { - ignoreTrees.Insert(*node.Subtree) - } - - if !ignore { - allNodesIgnored = false - } - - ignore, err = walk(ctx, repo, p, *node.Subtree, subtree, ignoreTrees, walkFn) + err = walk(ctx, repo, p, *node.Subtree, subtree, visitor) if err != nil { - return false, err - } - - if ignore { - ignoreTrees.Insert(*node.Subtree) - } - - if !ignore { - allNodesIgnored = false + return err } } - return allNodesIgnored, nil + if visitor.LeaveDir != nil { + visitor.LeaveDir(prefix) + } + + return nil } diff --git a/internal/walker/walker_test.go b/internal/walker/walker_test.go index 54cc69792..0f0009107 100644 --- a/internal/walker/walker_test.go +++ b/internal/walker/walker_test.go @@ -93,28 +93,32 @@ func (t TreeMap) Connections() uint { // checkFunc returns a function suitable for walking the tree to check // something, and a function which will check the final result. -type checkFunc func(t testing.TB) (walker WalkFunc, final func(testing.TB)) +type checkFunc func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) // checkItemOrder ensures that the order of the 'path' arguments is the one passed in as 'want'. func checkItemOrder(want []string) checkFunc { pos := 0 - return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { - walker = func(treeID restic.ID, path string, node *restic.Node, err error) (bool, error) { + return func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) { + walker = func(treeID restic.ID, path string, node *restic.Node, err error) error { if err != nil { t.Errorf("error walking %v: %v", path, err) - return false, err + return err } if pos >= len(want) { t.Errorf("additional unexpected path found: %v", path) - return false, nil + return nil } if path != want[pos] { t.Errorf("wrong path found, want %q, got %q", want[pos], path) } pos++ - return false, nil + return nil + } + + leaveDir = func(path string) { + _ = walker(restic.ID{}, "leave: "+path, nil, nil) } final = func(t testing.TB) { @@ -123,30 +127,30 @@ func checkItemOrder(want []string) checkFunc { } } - return walker, final + return walker, leaveDir, final } } // checkParentTreeOrder ensures that the order of the 'parentID' arguments is the one passed in as 'want'. func checkParentTreeOrder(want []string) checkFunc { pos := 0 - return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { - walker = func(treeID restic.ID, path string, node *restic.Node, err error) (bool, error) { + return func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) { + walker = func(treeID restic.ID, path string, node *restic.Node, err error) error { if err != nil { t.Errorf("error walking %v: %v", path, err) - return false, err + return err } if pos >= len(want) { t.Errorf("additional unexpected parent tree ID found: %v", treeID) - return false, nil + return nil } if treeID.String() != want[pos] { t.Errorf("wrong parent tree ID found, want %q, got %q", want[pos], treeID.String()) } pos++ - return false, nil + return nil } final = func(t testing.TB) { @@ -155,7 +159,7 @@ func checkParentTreeOrder(want []string) checkFunc { } } - return walker, final + return walker, nil, final } } @@ -164,16 +168,16 @@ func checkParentTreeOrder(want []string) checkFunc { func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc { var pos int - return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { - walker = func(treeID restic.ID, path string, node *restic.Node, err error) (bool, error) { + return func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) { + walker = func(treeID restic.ID, path string, node *restic.Node, err error) error { if err != nil { t.Errorf("error walking %v: %v", path, err) - return false, err + return err } if pos >= len(wantPaths) { t.Errorf("additional unexpected path found: %v", path) - return false, nil + return nil } if path != wantPaths[pos] { @@ -182,10 +186,14 @@ func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc { pos++ if _, ok := skipFor[path]; ok { - return false, ErrSkipNode + return ErrSkipNode } - return false, nil + return nil + } + + leaveDir = func(path string) { + _ = walker(restic.ID{}, "leave: "+path, nil, nil) } final = func(t testing.TB) { @@ -194,47 +202,7 @@ func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc { } } - return walker, final - } -} - -// checkIgnore returns ErrSkipNode if path is in skipFor and sets ignore according -// to ignoreFor. It checks that the paths the walk func is called for are exactly -// the ones in wantPaths. -func checkIgnore(skipFor map[string]struct{}, ignoreFor map[string]bool, wantPaths []string) checkFunc { - var pos int - - return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { - walker = func(treeID restic.ID, path string, node *restic.Node, err error) (bool, error) { - if err != nil { - t.Errorf("error walking %v: %v", path, err) - return false, err - } - - if pos >= len(wantPaths) { - t.Errorf("additional unexpected path found: %v", path) - return ignoreFor[path], nil - } - - if path != wantPaths[pos] { - t.Errorf("wrong path found, want %q, got %q", wantPaths[pos], path) - } - pos++ - - if _, ok := skipFor[path]; ok { - return ignoreFor[path], ErrSkipNode - } - - return ignoreFor[path], nil - } - - final = func(t testing.TB) { - if pos != len(wantPaths) { - t.Errorf("wrong number of paths returned, want %d, got %d", len(wantPaths), pos) - } - } - - return walker, final + return walker, leaveDir, final } } @@ -256,6 +224,8 @@ func TestWalker(t *testing.T) { "/foo", "/subdir", "/subdir/subfile", + "leave: /subdir", + "leave: /", }), checkParentTreeOrder([]string{ "a760536a8fd64dd63f8dd95d85d788d71fd1bee6828619350daf6959dcb499a0", // tree / @@ -270,16 +240,14 @@ func TestWalker(t *testing.T) { "/", "/foo", "/subdir", + "leave: /", }, ), - checkIgnore( - map[string]struct{}{}, map[string]bool{ - "/subdir": true, + checkSkipFor( + map[string]struct{}{ + "/": {}, }, []string{ "/", - "/foo", - "/subdir", - "/subdir/subfile", }, ), }, @@ -303,10 +271,14 @@ func TestWalker(t *testing.T) { "/foo", "/subdir1", "/subdir1/subfile1", + "leave: /subdir1", "/subdir2", "/subdir2/subfile2", "/subdir2/subsubdir2", "/subdir2/subsubdir2/subsubfile3", + "leave: /subdir2/subsubdir2", + "leave: /subdir2", + "leave: /", }), checkParentTreeOrder([]string{ "7a0e59b986cc83167d9fbeeefc54e4629770124c5825d391f7ee0598667fcdf1", // tree / @@ -329,6 +301,9 @@ func TestWalker(t *testing.T) { "/subdir2/subfile2", "/subdir2/subsubdir2", "/subdir2/subsubdir2/subsubfile3", + "leave: /subdir2/subsubdir2", + "leave: /subdir2", + "leave: /", }, ), checkSkipFor( @@ -342,6 +317,8 @@ func TestWalker(t *testing.T) { "/subdir2", "/subdir2/subfile2", "/subdir2/subsubdir2", + "leave: /subdir2", + "leave: /", }, ), checkSkipFor( @@ -350,6 +327,7 @@ func TestWalker(t *testing.T) { }, []string{ "/", "/foo", + "leave: /", }, ), }, @@ -382,15 +360,19 @@ func TestWalker(t *testing.T) { "/subdir1/subfile1", "/subdir1/subfile2", "/subdir1/subfile3", + "leave: /subdir1", "/subdir2", "/subdir2/subfile1", "/subdir2/subfile2", "/subdir2/subfile3", + "leave: /subdir2", "/subdir3", "/subdir3/subfile1", "/subdir3/subfile2", "/subdir3/subfile3", + "leave: /subdir3", "/zzz other", + "leave: /", }), checkParentTreeOrder([]string{ "c2efeff7f217a4dfa12a16e8bb3cefedd37c00873605c29e5271c6061030672f", // tree / @@ -409,81 +391,6 @@ func TestWalker(t *testing.T) { "57ee8960c7a86859b090a76e5d013f83d10c0ce11d5460076ca8468706f784ab", // tree /subdir3 "c2efeff7f217a4dfa12a16e8bb3cefedd37c00873605c29e5271c6061030672f", // tree / }), - checkIgnore( - map[string]struct{}{ - "/subdir1": {}, - }, map[string]bool{ - "/subdir1": true, - }, []string{ - "/", - "/foo", - "/subdir1", - "/zzz other", - }, - ), - checkIgnore( - map[string]struct{}{}, map[string]bool{ - "/subdir1": true, - }, []string{ - "/", - "/foo", - "/subdir1", - "/subdir1/subfile1", - "/subdir1/subfile2", - "/subdir1/subfile3", - "/zzz other", - }, - ), - checkIgnore( - map[string]struct{}{ - "/subdir2": {}, - }, map[string]bool{ - "/subdir2": true, - }, []string{ - "/", - "/foo", - "/subdir1", - "/subdir1/subfile1", - "/subdir1/subfile2", - "/subdir1/subfile3", - "/subdir2", - "/zzz other", - }, - ), - checkIgnore( - map[string]struct{}{}, map[string]bool{ - "/subdir1/subfile1": true, - "/subdir1/subfile2": true, - "/subdir1/subfile3": true, - }, []string{ - "/", - "/foo", - "/subdir1", - "/subdir1/subfile1", - "/subdir1/subfile2", - "/subdir1/subfile3", - "/zzz other", - }, - ), - checkIgnore( - map[string]struct{}{}, map[string]bool{ - "/subdir2/subfile1": true, - "/subdir2/subfile2": true, - "/subdir2/subfile3": true, - }, []string{ - "/", - "/foo", - "/subdir1", - "/subdir1/subfile1", - "/subdir1/subfile2", - "/subdir1/subfile3", - "/subdir2", - "/subdir2/subfile1", - "/subdir2/subfile2", - "/subdir2/subfile3", - "/zzz other", - }, - ), }, }, { @@ -503,45 +410,23 @@ func TestWalker(t *testing.T) { checkItemOrder([]string{ "/", "/subdir1", + "leave: /subdir1", "/subdir2", + "leave: /subdir2", "/subdir3", "/subdir3/file", + "leave: /subdir3", "/subdir4", "/subdir4/file", + "leave: /subdir4", "/subdir5", + "leave: /subdir5", "/subdir6", + "leave: /subdir6", + "leave: /", }), }, }, - { - tree: TestTree{ - "subdir1": TestTree{}, - "subdir2": TestTree{}, - "subdir3": TestTree{ - "file": TestFile{}, - }, - "subdir4": TestTree{}, - "subdir5": TestTree{ - "file": TestFile{}, - }, - "subdir6": TestTree{}, - }, - checks: []checkFunc{ - checkIgnore( - map[string]struct{}{}, map[string]bool{ - "/subdir2": true, - }, []string{ - "/", - "/subdir1", - "/subdir2", - "/subdir3", - "/subdir3/file", - "/subdir5", - "/subdir5/file", - }, - ), - }, - }, } for _, test := range tests { @@ -552,8 +437,11 @@ func TestWalker(t *testing.T) { ctx, cancel := context.WithCancel(context.TODO()) defer cancel() - fn, last := check(t) - err := Walk(ctx, repo, root, restic.NewIDSet(), fn) + fn, leaveDir, last := check(t) + err := Walk(ctx, repo, root, WalkVisitor{ + ProcessNode: fn, + LeaveDir: leaveDir, + }) if err != nil { t.Error(err) }