diff --git a/.github/ISSUE_TEMPLATE/Bug.md b/.github/ISSUE_TEMPLATE/Bug.md index b0741e3f8..532983773 100644 --- a/.github/ISSUE_TEMPLATE/Bug.md +++ b/.github/ISSUE_TEMPLATE/Bug.md @@ -32,23 +32,30 @@ Output of `restic version` -------------------------- -How did you run restic exactly? -------------------------------- + +What backend/service did you use to store the repository? +--------------------------------------------------------- + + + +Problem description / Steps to reproduce +---------------------------------------- -What backend/server/service did you use to store the repository? ----------------------------------------------------------------- - - Expected behavior ----------------- @@ -65,22 +72,12 @@ In this section, please try to concentrate on observations, so only describe what you observed directly. --> -Steps to reproduce the behavior -------------------------------- - - - Do you have any idea what may have caused this? ----------------------------------------------- - - -Do you have an idea how to solve the issue? -------------------------------------------- - + Did restic help you today? Did it make you happy in any way? diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 43c427109..a19767849 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -22,10 +22,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + uses: docker/login-action@3d58c274f17dffee475a5520cbe67f0a882c4dbb with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -42,10 +42,17 @@ jobs: type=semver,pattern={{major}}.{{minor}} - name: Set up QEMU - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 + + - name: Ensure consistent binaries + run: | + echo "removing git directory for consistency with release binaries" + rm -rf .git + # remove VCS information from release builds, keep VCS for nightly builds on master + if: github.ref != 'refs/heads/master' - name: Build and push Docker image uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9d88a6e94..1aabcedaf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ permissions: contents: read env: - latest_go: "1.20.x" + latest_go: "1.21.x" GO111MODULE: on jobs: @@ -23,18 +23,18 @@ jobs: # list of jobs to run: include: - job_name: Windows - go: 1.20.x + go: 1.21.x os: windows-latest test_smb: true - job_name: macOS - go: 1.20.x + go: 1.21.x os: macOS-latest test_fuse: false test_smb: true - job_name: Linux - go: 1.20.x + go: 1.21.x os: ubuntu-latest test_cloud_backends: true test_fuse: true @@ -42,20 +42,20 @@ jobs: check_changelog: true - job_name: Linux (race) - go: 1.20.x + go: 1.21.x os: ubuntu-latest test_fuse: true test_smb: true test_opts: "-race" - job_name: Linux - go: 1.19.x + go: 1.20.x os: ubuntu-latest test_fuse: true test_smb: true - job_name: Linux - go: 1.18.x + go: 1.19.x os: ubuntu-latest test_fuse: true test_smb: true @@ -252,7 +252,7 @@ jobs: if: matrix.os == 'windows-latest' - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build with build.go run: | @@ -349,7 +349,7 @@ jobs: go-version: ${{ env.latest_go }} - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Cross-compile for subset ${{ matrix.subset }} run: | @@ -367,13 +367,13 @@ jobs: go-version: ${{ env.latest_go }} - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.52.2 + version: v1.55.2 args: --verbose --timeout 5m # only run golangci-lint for pull requests, otherwise ALL hints get @@ -387,12 +387,27 @@ jobs: go mod tidy git diff --exit-code go.mod go.sum + analyze: + name: Analyze results + needs: [test, cross_compile, lint] + if: always() + + permissions: # no need to access code + contents: none + + runs-on: ubuntu-latest + steps: + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe + with: + jobs: ${{ toJSON(needs) }} + docker: name: docker runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Docker meta id: meta @@ -412,10 +427,10 @@ jobs: type=sha - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Build and push id: docker_build diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..acec8519b --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,22 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build HTMLZip +formats: + - htmlzip + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: doc/conf.py + +# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: doc/requirements.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index a502d49e2..c999b090d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,530 @@ -Changelog for restic 0.15.2 (2023-04-24) -======================================= +# Table of Contents +* [Changelog for 0.16.2](#changelog-for-restic-0162-2023-10-29) +* [Changelog for 0.16.1](#changelog-for-restic-0161-2023-10-24) +* [Changelog for 0.16.0](#changelog-for-restic-0160-2023-07-31) +* [Changelog for 0.15.2](#changelog-for-restic-0152-2023-04-24) +* [Changelog for 0.15.1](#changelog-for-restic-0151-2023-01-30) +* [Changelog for 0.15.0](#changelog-for-restic-0150-2023-01-12) +* [Changelog for 0.14.0](#changelog-for-restic-0140-2022-08-25) +* [Changelog for 0.13.0](#changelog-for-restic-0130-2022-03-26) +* [Changelog for 0.12.1](#changelog-for-restic-0121-2021-08-03) +* [Changelog for 0.12.0](#changelog-for-restic-0120-2021-02-14) +* [Changelog for 0.11.0](#changelog-for-restic-0110-2020-11-05) +* [Changelog for 0.10.0](#changelog-for-restic-0100-2020-09-19) +* [Changelog for 0.9.6](#changelog-for-restic-096-2019-11-22) +* [Changelog for 0.9.5](#changelog-for-restic-095-2019-04-23) +* [Changelog for 0.9.4](#changelog-for-restic-094-2019-01-06) +* [Changelog for 0.9.3](#changelog-for-restic-093-2018-10-13) +* [Changelog for 0.9.2](#changelog-for-restic-092-2018-08-06) +* [Changelog for 0.9.1](#changelog-for-restic-091-2018-06-10) +* [Changelog for 0.9.0](#changelog-for-restic-090-2018-05-21) +* [Changelog for 0.8.3](#changelog-for-restic-083-2018-02-26) +* [Changelog for 0.8.2](#changelog-for-restic-082-2018-02-17) +* [Changelog for 0.8.1](#changelog-for-restic-081-2017-12-27) +* [Changelog for 0.8.0](#changelog-for-restic-080-2017-11-26) +* [Changelog for 0.7.3](#changelog-for-restic-073-2017-09-20) +* [Changelog for 0.7.2](#changelog-for-restic-072-2017-09-13) +* [Changelog for 0.7.1](#changelog-for-restic-071-2017-07-22) +* [Changelog for 0.7.0](#changelog-for-restic-070-2017-07-01) +* [Changelog for 0.6.1](#changelog-for-restic-061-2017-06-01) +* [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29) + + +# Changelog for restic 0.16.2 (2023-10-29) +The following sections list the changes in restic 0.16.2 relevant to +restic users. The changes are ordered by importance. + +## Summary + + * Fix #4540: Restore ARMv5 support for ARM binaries + * Fix #4545: Repair documentation build on Read the Docs + +## Details + + * Bugfix #4540: Restore ARMv5 support for ARM binaries + + The official release binaries for restic 0.16.1 were accidentally built to require ARMv7. The + build process is now updated to restore support for ARMv5. + + Please note that restic 0.17.0 will drop support for ARMv5 and require at least ARMv6. + + https://github.com/restic/restic/issues/4540 + + * Bugfix #4545: Repair documentation build on Read the Docs + + For restic 0.16.1, no documentation was available at https://restic.readthedocs.io/ . + + The documentation build process is now updated to work again. + + https://github.com/restic/restic/pull/4545 + + +# Changelog for restic 0.16.1 (2023-10-24) +The following sections list the changes in restic 0.16.1 relevant to +restic users. The changes are ordered by importance. + +## Summary + + * Fix #4513: Make `key list` command honor `--no-lock` + * Fix #4516: Do not try to load password on command line autocomplete + * Fix #4523: Update zstd library to fix possible data corruption at max. compression + * Chg #4532: Update dependencies and require Go 1.19 or newer + * Enh #229: Show progress bar while loading the index + * Enh #4128: Automatically set `GOMAXPROCS` in resource-constrained containers + * Enh #4480: Allow setting REST password and username via environment variables + * Enh #4511: Include inode numbers in JSON output for `find` and `ls` commands + * Enh #4519: Add config option to set SFTP command arguments + +## Details + + * Bugfix #4513: Make `key list` command honor `--no-lock` + + The `key list` command now supports the `--no-lock` options. This allows determining which + keys a repo can be accessed by without the need for having write access (e.g., read-only sftp + access, filesystem snapshot). + + https://github.com/restic/restic/issues/4513 + https://github.com/restic/restic/pull/4514 + + * Bugfix #4516: Do not try to load password on command line autocomplete + + The command line autocompletion previously tried to load the repository password. This could + cause the autocompletion not to work. Now, this step gets skipped. + + https://github.com/restic/restic/issues/4516 + https://github.com/restic/restic/pull/4526 + + * Bugfix #4523: Update zstd library to fix possible data corruption at max. compression + + In restic 0.16.0, backups where the compression level was set to `max` (using `--compression + max`) could in rare and very specific circumstances result in data corruption due to a bug in the + library used for compressing data. + + Restic now uses the latest version of the library used to compress data, which includes a fix for + this issue. Please note that the `auto` compression level (which restic uses by default) was + never affected, and even if you used `max` compression, chances of being affected by this issue + were very small. + + To check a repository for any corruption, run `restic check --read-data`. This will download + and verify the whole repository and can be used at any time to completely verify the integrity of + a repository. If the `check` command detects anomalies, follow the suggested steps. + + To simplify any needed repository repair and minimize data loss, there is also a new and + experimental `repair packs` command that salvages all valid data from the affected pack files + (see `restic help repair packs` for more information). + + https://github.com/restic/restic/issues/4523 + https://github.com/restic/restic/pull/4530 + + * Change #4532: Update dependencies and require Go 1.19 or newer + + We have updated all dependencies. Since some libraries require newer Go standard library + features, support for Go 1.18 has been dropped, which means that restic now requires at least Go + 1.19 to build. + + https://github.com/restic/restic/pull/4532 + https://github.com/restic/restic/pull/4533 + + * Enhancement #229: Show progress bar while loading the index + + Restic did not provide any feedback while loading index files. Now, there is a progress bar that + shows the index loading progress. + + https://github.com/restic/restic/issues/229 + https://github.com/restic/restic/pull/4419 + + * Enhancement #4128: Automatically set `GOMAXPROCS` in resource-constrained containers + + When running restic in a Linux container with CPU-usage limits, restic now automatically + adjusts `GOMAXPROCS`. This helps to reduce the memory consumption on hosts with many CPU + cores. + + https://github.com/restic/restic/issues/4128 + https://github.com/restic/restic/pull/4485 + https://github.com/restic/restic/pull/4531 + + * Enhancement #4480: Allow setting REST password and username via environment variables + + Previously, it was only possible to specify the REST-server username and password in the + repository URL, or by using the `--repository-file` option. This meant it was not possible to + use authentication in contexts where the repository URL is stored in publicly accessible way. + + Restic now allows setting the username and password using the `RESTIC_REST_USERNAME` and + `RESTIC_REST_PASSWORD` variables. + + https://github.com/restic/restic/pull/4480 + + * Enhancement #4511: Include inode numbers in JSON output for `find` and `ls` commands + + Restic used to omit the inode numbers in the JSON messages emitted for nodes by the `ls` command + as well as for matches by the `find` command. It now includes those values whenever they are + available. + + https://github.com/restic/restic/pull/4511 + + * Enhancement #4519: Add config option to set SFTP command arguments + + When using the `sftp` backend, scenarios where a custom identity file was needed for the SSH + connection, required the full command to be specified: `-o sftp.command='ssh + user@host:port -i /ssh/my_private_key -s sftp'` + + Now, the `-o sftp.args=...` option can be passed to restic to specify custom arguments for the + SSH command executed by the SFTP backend. This simplifies the above example to `-o + sftp.args='-i /ssh/my_private_key'`. + + https://github.com/restic/restic/issues/4241 + https://github.com/restic/restic/pull/4519 + + +# Changelog for restic 0.16.0 (2023-07-31) +The following sections list the changes in restic 0.16.0 relevant to +restic users. The changes are ordered by importance. + +## Summary + + * Fix #2565: Support "unlimited" in `forget --keep-*` options + * Fix #3311: Support non-UTF8 paths as symlink target + * Fix #4199: Avoid lock refresh issues on slow network connections + * Fix #4274: Improve lock refresh handling after standby + * Fix #4319: Correctly clean up status bar output of the `backup` command + * Fix #4333: `generate` and `init` no longer silently ignore unexpected arguments + * Fix #4400: Ignore missing folders in `rest` backend + * Chg #4176: Fix JSON message type of `scan_finished` for the `backup` command + * Chg #4201: Require Go 1.20 for Solaris builds + * Enh #426: Show progress bar during restore + * Enh #719: Add `--retry-lock` option + * Enh #1495: Sort snapshots by timestamp in `restic find` + * Enh #1759: Add `repair index` and `repair snapshots` commands + * Enh #1926: Allow certificate paths to be passed through environment variables + * Enh #2359: Provide multi-platform Docker images + * Enh #2468: Add support for non-global Azure clouds + * Enh #2679: Reduce file fragmentation for local backend + * Enh #3328: Reduce memory usage by up to 25% + * Enh #3397: Improve accuracy of ETA displayed during backup + * Enh #3624: Keep oldest snapshot when there are not enough snapshots + * Enh #3698: Add support for Managed / Workload Identity to `azure` backend + * Enh #3871: Support `:` syntax to select subfolders + * Enh #3941: Support `--group-by` for backup parent selection + * Enh #4130: Cancel current command if cache becomes unusable + * Enh #4159: Add `--human-readable` option to `ls` and `find` commands + * Enh #4188: Include restic version in snapshot metadata + * Enh #4220: Add `jq` binary to Docker image + * Enh #4226: Allow specifying region of new buckets in the `gs` backend + * Enh #4375: Add support for extended attributes on symlinks + +## Details + + * Bugfix #2565: Support "unlimited" in `forget --keep-*` options + + Restic would previously forget snapshots that should have been kept when a negative value was + passed to the `--keep-*` options. Negative values are now forbidden. To keep all snapshots, + the special value `unlimited` is now supported. For example, `--keep-monthly unlimited` + will keep all monthly snapshots. + + https://github.com/restic/restic/issues/2565 + https://github.com/restic/restic/pull/4234 + + * Bugfix #3311: Support non-UTF8 paths as symlink target + + Earlier restic versions did not correctly `backup` and `restore` symlinks that contain a + non-UTF8 target. Note that this only affected systems that still use a non-Unicode encoding + for filesystem paths. + + The repository format is now extended to add support for such symlinks. Please note that + snapshots must have been created with at least restic version 0.16.0 for `restore` to + correctly handle non-UTF8 symlink targets when restoring them. + + https://github.com/restic/restic/issues/3311 + https://github.com/restic/restic/pull/3802 + + * Bugfix #4199: Avoid lock refresh issues on slow network connections + + On network connections with a low upload speed, backups and other operations could fail with + the error message `Fatal: failed to refresh lock in time`. + + This has now been fixed by reworking the lock refresh handling. + + https://github.com/restic/restic/issues/4199 + https://github.com/restic/restic/pull/4304 + + * Bugfix #4274: Improve lock refresh handling after standby + + If the restic process was stopped or the host running restic entered standby during a long + running operation such as a backup, this previously resulted in the operation failing with + `Fatal: failed to refresh lock in time`. + + This has now been fixed such that restic first checks whether it is safe to continue the current + operation and only throws an error if not. + + https://github.com/restic/restic/issues/4274 + https://github.com/restic/restic/pull/4374 + + * Bugfix #4319: Correctly clean up status bar output of the `backup` command + + Due to a regression in restic 0.15.2, the status bar of the `backup` command could leave some + output behind. This happened if filenames were printed that are wider than the current + terminal width. This has now been fixed. + + https://github.com/restic/restic/issues/4319 + https://github.com/restic/restic/pull/4318 + + * Bugfix #4333: `generate` and `init` no longer silently ignore unexpected arguments + + https://github.com/restic/restic/pull/4333 + + * Bugfix #4400: Ignore missing folders in `rest` backend + + If a repository accessed via the REST backend was missing folders, then restic would fail with + an error while trying to list the data in the repository. This has been now fixed. + + https://github.com/restic/rest-server/issues/235 + https://github.com/restic/restic/pull/4400 + + * Change #4176: Fix JSON message type of `scan_finished` for the `backup` command + + Restic incorrectly set the `message_type` of the `scan_finished` message to `status` + instead of `verbose_status`. This has now been corrected so that the messages report the + correct type. + + https://github.com/restic/restic/pull/4176 + + * Change #4201: Require Go 1.20 for Solaris builds + + Building restic on Solaris now requires Go 1.20, as the library used to access Azure uses the + mmap syscall, which is only available on Solaris starting from Go 1.20. All other platforms + however continue to build with Go 1.18. + + https://github.com/restic/restic/pull/4201 + + * Enhancement #426: Show progress bar during restore + + The `restore` command now shows a progress report while restoring files. + + Example: `[0:42] 5.76% 23 files 12.98 MiB, total 3456 files 23.54 GiB` + + JSON output is now also supported. + + https://github.com/restic/restic/issues/426 + https://github.com/restic/restic/issues/3413 + https://github.com/restic/restic/issues/3627 + https://github.com/restic/restic/pull/3991 + https://github.com/restic/restic/pull/4314 + https://forum.restic.net/t/progress-bar-for-restore/5210 + + * Enhancement #719: Add `--retry-lock` option + + This option allows specifying a duration for which restic will wait if the repository is + already locked. + + https://github.com/restic/restic/issues/719 + https://github.com/restic/restic/pull/2214 + https://github.com/restic/restic/pull/4107 + + * Enhancement #1495: Sort snapshots by timestamp in `restic find` + + The `find` command used to print snapshots in an arbitrary order. Restic now prints snapshots + sorted by timestamp. + + https://github.com/restic/restic/issues/1495 + https://github.com/restic/restic/pull/4409 + + * Enhancement #1759: Add `repair index` and `repair snapshots` commands + + The `rebuild-index` command has been renamed to `repair index`. The old name will still work, + but is deprecated. + + When a snapshot was damaged, the only option up to now was to completely forget the snapshot, + even if only some unimportant files in it were damaged and other files were still fine. + + Restic now has a `repair snapshots` command, which can salvage any non-damaged files and parts + of files in the snapshots by removing damaged directories and missing file contents. Please + note that the damaged data may still be lost and see the "Troubleshooting" section in the + documentation for more details. + + https://github.com/restic/restic/issues/1759 + https://github.com/restic/restic/issues/1714 + https://github.com/restic/restic/issues/1798 + https://github.com/restic/restic/issues/2334 + https://github.com/restic/restic/pull/2876 + https://forum.restic.net/t/corrupted-repo-how-to-repair/799 + https://forum.restic.net/t/recovery-options-for-damaged-repositories/1571 + + * Enhancement #1926: Allow certificate paths to be passed through environment variables + + Restic will now read paths to certificates from the environment variables `RESTIC_CACERT` or + `RESTIC_TLS_CLIENT_CERT` if `--cacert` or `--tls-client-cert` are not specified. + + https://github.com/restic/restic/issues/1926 + https://github.com/restic/restic/pull/4384 + + * Enhancement #2359: Provide multi-platform Docker images + + The official Docker images are now built for the architectures linux/386, linux/amd64, + linux/arm and linux/arm64. + + As an alternative to the Docker Hub, the Docker images are also available on ghcr.io, the GitHub + Container Registry. + + https://github.com/restic/restic/issues/2359 + https://github.com/restic/restic/issues/4269 + https://github.com/restic/restic/pull/4364 + + * Enhancement #2468: Add support for non-global Azure clouds + + The `azure` backend previously only supported storages using the global domain + `core.windows.net`. This meant that backups to other domains such as Azure China + (`core.chinacloudapi.cn`) or Azure Germany (`core.cloudapi.de`) were not supported. + Restic now allows overriding the global domain using the environment variable + `AZURE_ENDPOINT_SUFFIX`. + + https://github.com/restic/restic/issues/2468 + https://github.com/restic/restic/pull/4387 + + * Enhancement #2679: Reduce file fragmentation for local backend + + Before this change, local backend files could become fragmented. Now restic will try to + preallocate space for pack files to avoid their fragmentation. + + https://github.com/restic/restic/issues/2679 + https://github.com/restic/restic/pull/3261 + + * Enhancement #3328: Reduce memory usage by up to 25% + + The in-memory index has been optimized to be more garbage collection friendly. Restic now + defaults to `GOGC=50` to run the Go garbage collector more frequently. + + https://github.com/restic/restic/issues/3328 + https://github.com/restic/restic/pull/4352 + https://github.com/restic/restic/pull/4353 + + * Enhancement #3397: Improve accuracy of ETA displayed during backup + + Restic's `backup` command displayed an ETA that did not adapt when the rate of progress made + during the backup changed during the course of the backup. + + Restic now uses recent progress when computing the ETA. It is important to realize that the + estimate may still be wrong, because restic cannot predict the future, but the hope is that the + ETA will be more accurate in most cases. + + https://github.com/restic/restic/issues/3397 + https://github.com/restic/restic/pull/3563 + + * Enhancement #3624: Keep oldest snapshot when there are not enough snapshots + + The `forget` command now additionally preserves the oldest snapshot if fewer snapshots than + allowed by the `--keep-*` parameters would otherwise be kept. This maximizes the amount of + history kept within the specified limits. + + https://github.com/restic/restic/issues/3624 + https://github.com/restic/restic/pull/4366 + https://forum.restic.net/t/keeping-yearly-snapshots-policy-when-backup-began-during-the-year/4670/2 + + * Enhancement #3698: Add support for Managed / Workload Identity to `azure` backend + + Restic now additionally supports authenticating to Azure using Workload Identity or Managed + Identity credentials, which are automatically injected in several environments such as a + managed Kubernetes cluster. + + https://github.com/restic/restic/issues/3698 + https://github.com/restic/restic/pull/4029 + + * Enhancement #3871: Support `:` syntax to select subfolders + + Commands like `diff` or `restore` always worked with the full snapshot. This did not allow + comparing only a specific subfolder or only restoring that folder (`restore --include + subfolder` filters the restored files, but still creates the directories included in + `subfolder`). + + The commands `diff`, `dump`, `ls` and `restore` now support the `:` + syntax, where `snapshot` is the ID of a snapshot (or the string `latest`) and `subfolder` is a + path within the snapshot. The commands will then only work with the specified path of the + snapshot. The `subfolder` must be a path to a folder as returned by `ls`. Two examples: + + `restic restore -t target latest:/some/path` `restic diff 12345678:/some/path + 90abcef:/some/path` + + For debugging purposes, the `cat` command now supports `cat tree :` to + return the directory metadata for the given subfolder. + + https://github.com/restic/restic/issues/3871 + https://github.com/restic/restic/pull/4334 + + * Enhancement #3941: Support `--group-by` for backup parent selection + + Previously, the `backup` command by default selected the parent snapshot based on the + hostname and the backup targets. When the backup path list changed, the `backup` command was + unable to determine a suitable parent snapshot and had to read all files again. + + The new `--group-by` option for the `backup` command allows filtering snapshots for the + parent selection by `host`, `paths` and `tags`. It defaults to `host,paths` which selects the + latest snapshot with hostname and paths matching those of the backup run. This matches the + behavior of prior restic versions. + + The new `--group-by` option should be set to the same value as passed to `forget --group-by`. + + https://github.com/restic/restic/issues/3941 + https://github.com/restic/restic/pull/4081 + + * Enhancement #4130: Cancel current command if cache becomes unusable + + If the cache directory was removed or ran out of space while restic was running, this would + previously cause further caching attempts to fail and thereby drastically slow down the + command execution. Now, the currently running command is instead canceled. + + https://github.com/restic/restic/issues/4130 + https://github.com/restic/restic/pull/4166 + + * Enhancement #4159: Add `--human-readable` option to `ls` and `find` commands + + Previously, when using the `-l` option with the `ls` and `find` commands, the displayed size + was always in bytes, without an option for a more human readable format such as MiB or GiB. + + The new `--human-readable` option will convert longer size values into more human friendly + values with an appropriate suffix depending on the output size. For example, a size of + `14680064` will be shown as `14.000 MiB`. + + https://github.com/restic/restic/issues/4159 + https://github.com/restic/restic/pull/4351 + + * Enhancement #4188: Include restic version in snapshot metadata + + The restic version used to backup a snapshot is now included in its metadata and shown when + inspecting a snapshot using `restic cat snapshot ` or `restic snapshots + --json`. + + https://github.com/restic/restic/issues/4188 + https://github.com/restic/restic/pull/4378 + + * Enhancement #4220: Add `jq` binary to Docker image + + The Docker image now contains `jq`, which can be useful to process JSON data output by restic. + + https://github.com/restic/restic/pull/4220 + + * Enhancement #4226: Allow specifying region of new buckets in the `gs` backend + + Previously, buckets used by the Google Cloud Storage backend would always get created in the + "us" region. It is now possible to specify the region where a bucket should be created by using + the `-o gs.region=us` option. + + https://github.com/restic/restic/pull/4226 + + * Enhancement #4375: Add support for extended attributes on symlinks + + Restic now supports extended attributes on symlinks when backing up, restoring, or + FUSE-mounting snapshots. This includes, for example, the `security.selinux` xattr on Linux + distributions that use SELinux. + + https://github.com/restic/restic/issues/4375 + https://github.com/restic/restic/pull/4379 + + +# Changelog for restic 0.15.2 (2023-04-24) The following sections list the changes in restic 0.15.2 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Sec #4275: Update golang.org/x/net to address CVE-2022-41723 * Fix #2260: Sanitize filenames printed by `backup` during processing @@ -15,8 +534,7 @@ Summary * Enh #4180: Add release binaries for riscv64 architecture on Linux * Enh #4219: Upgrade Minio to version 7.0.49 -Details -------- +## Details * Security #4275: Update golang.org/x/net to address CVE-2022-41723 @@ -87,14 +605,11 @@ Details https://github.com/restic/restic/pull/4219 -Changelog for restic 0.15.1 (2023-01-30) -======================================= - +# Changelog for restic 0.15.1 (2023-01-30) The following sections list the changes in restic 0.15.1 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #3750: Remove `b2_download_file_by_name: 404` warning from B2 backend * Fix #4147: Make `prune --quiet` not print progress bar @@ -102,8 +617,7 @@ Summary * Fix #4167: Add missing ETA in `backup` progress bar * Enh #4143: Ignore empty lock files -Details -------- +## Details * Bugfix #3750: Remove `b2_download_file_by_name: 404` warning from B2 backend @@ -162,14 +676,11 @@ Details https://github.com/restic/restic/pull/4152 -Changelog for restic 0.15.0 (2023-01-12) -======================================= - +# Changelog for restic 0.15.0 (2023-01-12) The following sections list the changes in restic 0.15.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #2015: Make `mount` return exit code 0 after receiving Ctrl-C / SIGINT * Fix #2578: Make `restore` replace existing symlinks @@ -211,8 +722,7 @@ Summary * Enh #3943: Ignore additional/unknown files in repository * Enh #3955: Improve `backup` performance for small files -Details -------- +## Details * Bugfix #2015: Make `mount` return exit code 0 after receiving Ctrl-C / SIGINT @@ -643,14 +1153,11 @@ Details https://github.com/restic/restic/pull/3955 -Changelog for restic 0.14.0 (2022-08-25) -======================================= - +# Changelog for restic 0.14.0 (2022-08-25) The following sections list the changes in restic 0.14.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #2248: Support `self-update` on Windows * Fix #3428: List snapshots in backend at most once to resolve snapshot IDs @@ -686,8 +1193,7 @@ Summary * Enh #3819: Validate include/exclude patterns before restoring * Enh #3837: Improve SFTP repository initialization over slow links -Details -------- +## Details * Bugfix #2248: Support `self-update` on Windows @@ -1024,7 +1530,7 @@ Details Restic did not support limiting the IO concurrency / number of connections for accessing repositories stored using the local or SFTP backends. The number of connections is now limited - as for other backends, and can be configured via the the `-o local.connections=2` and `-o + as for other backends, and can be configured via the `-o local.connections=2` and `-o sftp.connections=5` options. This ensures that restic does not overwhelm the backend with concurrent IO operations. @@ -1093,14 +1599,11 @@ Details https://github.com/restic/restic/pull/3840 -Changelog for restic 0.13.0 (2022-03-26) -======================================= - +# Changelog for restic 0.13.0 (2022-03-26) The following sections list the changes in restic 0.13.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1106: Never lock repository for `list locks` * Fix #2345: Make cache crash-resistant and usable by multiple concurrent processes @@ -1137,8 +1640,7 @@ Summary * Enh #3542: Add file mode in symbolic notation to `ls --json` * Enh #3593: Improve `copy` performance by parallelizing IO -Details -------- +## Details * Bugfix #1106: Never lock repository for `list locks` @@ -1498,14 +2000,11 @@ Details https://github.com/restic/restic/pull/3593 -Changelog for restic 0.12.1 (2021-08-03) -======================================= - +# Changelog for restic 0.12.1 (2021-08-03) The following sections list the changes in restic 0.12.1 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #2742: Improve error handling for rclone and REST backend over HTTP2 * Fix #3111: Fix terminal output redirection for PowerShell @@ -1530,8 +2029,7 @@ Summary * Enh #3427: `find --pack` fallback to index if data file is missing * Enh #3456: Support filtering and specifying untagged snapshots -Details -------- +## Details * Bugfix #2742: Improve error handling for rclone and REST backend over HTTP2 @@ -1744,14 +2242,11 @@ Details https://github.com/restic/restic/pull/3457 -Changelog for restic 0.12.0 (2021-02-14) -======================================= - +# Changelog for restic 0.12.0 (2021-02-14) The following sections list the changes in restic 0.12.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1681: Make `mount` not create missing mount point directory * Fix #1800: Ignore `no data available` filesystem error during backup @@ -1789,8 +2284,7 @@ Summary * Enh #3250: Add several more error checks * Enh #3254: Enable HTTP/2 for backend connections -Details -------- +## Details * Bugfix #1681: Make `mount` not create missing mount point directory @@ -2183,7 +2677,7 @@ Details * Enhancement #3106: Parallelize scan of snapshot content in `copy` and `prune` The `copy` and `prune` commands used to traverse the directories of snapshots one by one to find - used data. This snapshot traversal is now parallized which can speed up this step several + used data. This snapshot traversal is now parallelized which can speed up this step several times. In addition the `check` command now reports how many snapshots have already been processed. @@ -2241,14 +2735,11 @@ Details https://github.com/restic/restic/pull/3254 -Changelog for restic 0.11.0 (2020-11-05) -======================================= - +# Changelog for restic 0.11.0 (2020-11-05) The following sections list the changes in restic 0.11.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1212: Restore timestamps and permissions on intermediate directories * Fix #1756: Mark repository files as read-only when using the local backend @@ -2266,8 +2757,7 @@ Summary * Enh #2969: Optimize check for unchanged files during backup * Enh #2978: Warn if parent snapshot cannot be loaded during backup -Details -------- +## Details * Bugfix #1212: Restore timestamps and permissions on intermediate directories @@ -2283,7 +2773,7 @@ Details * Bugfix #1756: Mark repository files as read-only when using the local backend - Files stored in a local repository were marked as writeable on the filesystem for non-Windows + Files stored in a local repository were marked as writable on the filesystem for non-Windows systems, which did not prevent accidental file modifications outside of restic. In addition, the local backend did not work with certain filesystems and network mounts which do not permit modifications of file permissions. @@ -2373,8 +2863,7 @@ Details an exclusive lock through a filesystem snapshot. Restic was unable to backup those files before. This update enables backing up these files. - This needs to be enabled explicitely using the --use-fs-snapshot option of the backup - command. + This needs to be enabled explicitly using the --use-fs-snapshot option of the backup command. https://github.com/restic/restic/issues/340 https://github.com/restic/restic/pull/2274 @@ -2416,14 +2905,11 @@ Details https://github.com/restic/restic/pull/2978 -Changelog for restic 0.10.0 (2020-09-19) -======================================= - +# Changelog for restic 0.10.0 (2020-09-19) The following sections list the changes in restic 0.10.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1863: Report correct number of directories processed by backup * Fix #2254: Fix tar issues when dumping `/` @@ -2470,8 +2956,7 @@ Summary * Enh #2840: Speed-up file deletion in forget, prune and rebuild-index * Enh #2858: Support filtering snapshots by tag and path in the stats command -Details -------- +## Details * Bugfix #1863: Report correct number of directories processed by backup @@ -2578,7 +3063,7 @@ Details * Bugfix #2668: Don't abort the stats command when data blobs are missing - Runing the stats command in the blobs-per-file mode on a repository with missing data blobs + Running the stats command in the blobs-per-file mode on a repository with missing data blobs previously resulted in a crash. https://github.com/restic/restic/pull/2668 @@ -2880,14 +3365,11 @@ Details https://forum.restic.net/t/stats-for-a-host-and-filtered-snapshots/3020 -Changelog for restic 0.9.6 (2019-11-22) -======================================= - +# Changelog for restic 0.9.6 (2019-11-22) The following sections list the changes in restic 0.9.6 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #2063: Allow absolute path for filename when backing up from stdin * Fix #2174: Save files with invalid timestamps @@ -2899,8 +3381,7 @@ Summary * Enh #2330: Make `--group-by` accept both singular and plural * Enh #2350: Add option to configure S3 region -Details -------- +## Details * Bugfix #2063: Allow absolute path for filename when backing up from stdin @@ -2953,7 +3434,7 @@ Details check will be disabled if the --ignore-inode flag was given. If this change causes problems for you, please open an issue, and we can look in to adding a - seperate flag to disable just the ctime check. + separate flag to disable just the ctime check. https://github.com/restic/restic/issues/2179 https://github.com/restic/restic/pull/2212 @@ -2983,14 +3464,11 @@ Details https://github.com/restic/restic/pull/2350 -Changelog for restic 0.9.5 (2019-04-23) -======================================= - +# Changelog for restic 0.9.5 (2019-04-23) The following sections list the changes in restic 0.9.5 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #2135: Return error when no bytes could be read from stdin * Fix #2181: Don't cancel timeout after 30 seconds for self-update @@ -3006,8 +3484,7 @@ Summary * Enh #2205: Add --ignore-inode option to backup cmd * Enh #2220: Add config option to set S3 storage class -Details -------- +## Details * Bugfix #2135: Return error when no bytes could be read from stdin @@ -3125,14 +3602,11 @@ Details https://github.com/restic/restic/pull/2220 -Changelog for restic 0.9.4 (2019-01-06) -======================================= - +# Changelog for restic 0.9.4 (2019-01-06) The following sections list the changes in restic 0.9.4 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1989: Google Cloud Storage: Respect bandwidth limit * Fix #2040: Add host name filter shorthand flag for `stats` command @@ -3146,8 +3620,7 @@ Summary * Enh #2094: Run command to get password * Enh #2097: Add key hinting -Details -------- +## Details * Bugfix #1989: Google Cloud Storage: Respect bandwidth limit @@ -3253,14 +3726,11 @@ Details https://github.com/restic/restic/issues/2097 -Changelog for restic 0.9.3 (2018-10-13) -======================================= - +# Changelog for restic 0.9.3 (2018-10-13) The following sections list the changes in restic 0.9.3 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1935: Remove truncated files from cache * Fix #1978: Do not return an error when the scanner is slower than backup @@ -3277,8 +3747,7 @@ Summary * Enh #1967: Use `--host` everywhere * Enh #2028: Display size of cache directories -Details -------- +## Details * Bugfix #1935: Remove truncated files from cache @@ -3325,7 +3794,7 @@ Details * Enhancement #1876: Display reason why forget keeps snapshots We've added a column to the list of snapshots `forget` keeps which details the reasons to keep a - particuliar snapshot. This makes debugging policies for forget much easier. Please remember + particular snapshot. This makes debugging policies for forget much easier. Please remember to always try things out with `--dry-run`! https://github.com/restic/restic/pull/1876 @@ -3424,14 +3893,11 @@ Details https://github.com/restic/restic/pull/2033 -Changelog for restic 0.9.2 (2018-08-06) -======================================= - +# Changelog for restic 0.9.2 (2018-08-06) The following sections list the changes in restic 0.9.2 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1854: Allow saving files/dirs on different fs with `--one-file-system` * Fix #1861: Fix case-insensitive search with restic find @@ -3445,8 +3911,7 @@ Summary * Enh #1901: Update the Backblaze B2 library * Enh #1906: Add support for B2 application keys -Details -------- +## Details * Bugfix #1854: Allow saving files/dirs on different fs with `--one-file-system` @@ -3546,14 +4011,11 @@ Details https://github.com/restic/restic/pull/1914 -Changelog for restic 0.9.1 (2018-06-10) -======================================= - +# Changelog for restic 0.9.1 (2018-06-10) The following sections list the changes in restic 0.9.1 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1801: Add limiting bandwidth to the rclone backend * Fix #1822: Allow uploading large files to MS Azure @@ -3561,8 +4023,7 @@ Summary * Fix #1833: Fix caching files on error * Fix #1834: Resolve deadlock -Details -------- +## Details * Bugfix #1801: Add limiting bandwidth to the rclone backend @@ -3613,14 +4074,11 @@ Details https://github.com/restic/restic/pull/1835 -Changelog for restic 0.9.0 (2018-05-21) -======================================= - +# Changelog for restic 0.9.0 (2018-05-21) The following sections list the changes in restic 0.9.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1608: Respect time stamp for new backup when reading from stdin * Fix #1652: Ignore/remove invalid lock files @@ -3638,12 +4096,11 @@ Summary * Enh #1665: Improve cache handling for `restic check` * Enh #1709: Improve messages `restic check` prints * Enh #1721: Add `cache` command to list cache dirs - * Enh #1735: Allow keeping a time range of snaphots + * Enh #1735: Allow keeping a time range of snapshots * Enh #1758: Allow saving OneDrive folders in Windows * Enh #1782: Use default AWS credentials chain for S3 backend -Details -------- +## Details * Bugfix #1608: Respect time stamp for new backup when reading from stdin @@ -3838,7 +4295,7 @@ Details https://github.com/restic/restic/issues/1721 https://github.com/restic/restic/pull/1749 - * Enhancement #1735: Allow keeping a time range of snaphots + * Enhancement #1735: Allow keeping a time range of snapshots We've added the `--keep-within` option to the `forget` command. It instructs restic to keep all snapshots within the given duration since the newest snapshot. For example, running @@ -3865,14 +4322,11 @@ Details https://github.com/restic/restic/pull/1782 -Changelog for restic 0.8.3 (2018-02-26) -======================================= - +# Changelog for restic 0.8.3 (2018-02-26) The following sections list the changes in restic 0.8.3 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1633: Fixed unexpected 'pack file cannot be listed' error * Fix #1638: Handle errors listing files in the backend @@ -3882,8 +4336,7 @@ Summary * Enh #1623: Don't check for presence of files in the backend before writing * Enh #1634: Upgrade B2 client library, reduce HTTP requests -Details -------- +## Details * Bugfix #1633: Fixed unexpected 'pack file cannot be listed' error @@ -3939,7 +4392,7 @@ Details HTTP) and returning an error when the file already exists. This is not accurate, the file could have been created between the HTTP request testing for it, - and when writing starts, so we've relaxed this requeriment, which saves one additional HTTP + and when writing starts, so we've relaxed this requirement, which saves one additional HTTP request per newly added file. https://github.com/restic/restic/pull/1623 @@ -3953,16 +4406,13 @@ Details https://github.com/restic/restic/pull/1634 -Changelog for restic 0.8.2 (2018-02-17) -======================================= - +# Changelog for restic 0.8.2 (2018-02-17) The following sections list the changes in restic 0.8.2 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary - * Fix #1506: Limit bandwith at the http.RoundTripper for HTTP based backends + * Fix #1506: Limit bandwidth at the http.RoundTripper for HTTP based backends * Fix #1512: Restore directory permissions as the last step * Fix #1528: Correctly create missing subdirs in data/ * Fix #1589: Complete intermediate index upload @@ -3980,10 +4430,9 @@ Summary * Enh #1579: Retry Backend.List() in case of errors * Enh #1584: Limit index file size -Details -------- +## Details - * Bugfix #1506: Limit bandwith at the http.RoundTripper for HTTP based backends + * Bugfix #1506: Limit bandwidth at the http.RoundTripper for HTTP based backends https://github.com/restic/restic/issues/1506 https://github.com/restic/restic/pull/1511 @@ -4036,8 +4485,8 @@ Details * Bugfix #1595: Backup: Remove bandwidth display This commit removes the bandwidth displayed during backup process. It is misleading and - seldomly correct, because it's neither the "read bandwidth" (only for the very first backup) - nor the "upload bandwidth". Many users are confused about (and rightly so), c.f. #1581, #1033, + seldom correct, because it's neither the "read bandwidth" (only for the very first backup) nor + the "upload bandwidth". Many users are confused about (and rightly so), c.f. #1581, #1033, #1591 We'll eventually replace this display with something more relevant when the new archiver code @@ -4126,14 +4575,11 @@ Details https://github.com/restic/restic/pull/1584 -Changelog for restic 0.8.1 (2017-12-27) -======================================= - +# Changelog for restic 0.8.1 (2017-12-27) The following sections list the changes in restic 0.8.1 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1454: Correct cache dir location for Windows and Darwin * Fix #1457: Improve s3 backend with DigitalOcean Spaces @@ -4143,8 +4589,7 @@ Summary * Enh #1436: Add code to detect old cache directories * Enh #1439: Improve cancellation logic -Details -------- +## Details * Bugfix #1454: Correct cache dir location for Windows and Darwin @@ -4202,14 +4647,11 @@ Details https://github.com/restic/restic/pull/1439 -Changelog for restic 0.8.0 (2017-11-26) -======================================= - +# Changelog for restic 0.8.0 (2017-11-26) The following sections list the changes in restic 0.8.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Sec #1445: Prevent writing outside the target directory during restore * Fix #1256: Re-enable workaround for S3 backend @@ -4231,8 +4673,7 @@ Summary * Enh #1353: Retry failed backend requests * Enh #1367: Allow comments in files read from via `--file-from` -Details -------- +## Details * Security #1445: Prevent writing outside the target directory during restore @@ -4306,7 +4747,7 @@ Details We've added a local cache for metadata so that restic doesn't need to load all metadata (snapshots, indexes, ...) from the repo each time it starts. By default the cache is active, but - there's a new global option `--no-cache` that can be used to disable the cache. By deafult, the + there's a new global option `--no-cache` that can be used to disable the cache. By default, the cache a standard cache folder for the OS, which can be overridden with `--cache-dir`. The cache will automatically populate, indexes and snapshots are saved as they are loaded. Cache directories for repos that haven't been used recently can automatically be removed by restic @@ -4392,7 +4833,7 @@ Details * Enhancement #1319: Make `check` print `no errors found` explicitly - The `check` command now explicetly prints `No errors were found` when no errors could be found. + The `check` command now explicitly prints `No errors were found` when no errors could be found. https://github.com/restic/restic/issues/1303 https://github.com/restic/restic/pull/1319 @@ -4410,19 +4851,15 @@ Details https://github.com/restic/restic/pull/1368 -Changelog for restic 0.7.3 (2017-09-20) -======================================= - +# Changelog for restic 0.7.3 (2017-09-20) The following sections list the changes in restic 0.7.3 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1246: List all files stored in Google Cloud Storage -Details -------- +## Details * Bugfix #1246: List all files stored in Google Cloud Storage @@ -4434,14 +4871,11 @@ Details https://github.com/restic/restic/pull/1247 -Changelog for restic 0.7.2 (2017-09-13) -======================================= - +# Changelog for restic 0.7.2 (2017-09-13) The following sections list the changes in restic 0.7.2 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1164: Make the `key remove` command behave as documented * Fix #1167: Do not create a local repo unless `init` is used @@ -4461,8 +4895,7 @@ Summary * Enh #1205: Allow specifying time/date for a backup with `--time` * Enh #1218: Add `--compact` to `snapshots` command -Details -------- +## Details * Bugfix #1164: Make the `key remove` command behave as documented @@ -4570,14 +5003,11 @@ Details https://github.com/restic/restic/pull/1223 -Changelog for restic 0.7.1 (2017-07-22) -======================================= - +# Changelog for restic 0.7.1 (2017-07-22) The following sections list the changes in restic 0.7.1 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1115: Fix `prune`, only include existing files in indexes * Enh #1055: Create subdirs below `data/` for local/sftp backends @@ -4587,8 +5017,7 @@ Summary * Enh #1081: Clarify semantic for `--tag` for the `forget` command * Enh #1082: Print stats on SIGINFO on Darwin and FreeBSD (ctrl+t) -Details -------- +## Details * Bugfix #1115: Fix `prune`, only include existing files in indexes @@ -4642,14 +5071,11 @@ Details https://github.com/restic/restic/pull/1082 -Changelog for restic 0.7.0 (2017-07-01) -======================================= - +# Changelog for restic 0.7.0 (2017-07-01) The following sections list the changes in restic 0.7.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #965: Switch to `default` repo layout for the s3 backend * Fix #1013: Switch back to using the high-level minio-go API for s3 @@ -4661,8 +5087,7 @@ Summary * Enh #1021: Detect invalid backend name and print error * Enh #1029: Remove invalid pack files when `prune` is run -Details -------- +## Details * Bugfix #965: Switch to `default` repo layout for the s3 backend @@ -4733,21 +5158,17 @@ Details https://github.com/restic/restic/pull/1036 -Changelog for restic 0.6.1 (2017-06-01) -======================================= - +# Changelog for restic 0.6.1 (2017-06-01) The following sections list the changes in restic 0.6.1 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Enh #974: Remove regular status reports * Enh #981: Remove temporary path from binary in `build.go` * Enh #985: Allow multiple parallel idle HTTP connections -Details -------- +## Details * Enhancement #974: Remove regular status reports @@ -4774,21 +5195,17 @@ Details https://github.com/restic/restic/pull/986 -Changelog for restic 0.6.0 (2017-05-29) -======================================= - +# Changelog for restic 0.6.0 (2017-05-29) The following sections list the changes in restic 0.6.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Enh #957: Make `forget` consistent * Enh #962: Improve memory and runtime for the s3 backend * Enh #966: Unify repository layout for all backends -Details -------- +## Details * Enhancement #957: Make `forget` consistent @@ -4824,5 +5241,3 @@ Details https://github.com/restic/restic/issues/965 https://github.com/restic/restic/pull/966 - - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 39a829337..4318a2107 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,7 +61,7 @@ uploading it somewhere or post only the parts that are really relevant. If restic gets stuck, please also include a stacktrace in the description. On non-Windows systems, you can send a SIGQUIT signal to restic or press `Ctrl-\` to achieve the same result. This causes restic to print a stacktrace -and then exit immediatelly. This will not damage your repository, however, +and then exit immediately. This will not damage your repository, however, it might be necessary to manually clean up stale lock files using `restic unlock`. diff --git a/README.md b/README.md index 8f72a0200..ad6b13cef 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ release. Instructions on how to do that are contained in the News ---- -You can follow the restic project on Twitter [@resticbackup](https://twitter.com/resticbackup) or by subscribing to +You can follow the restic project on Mastodon [@resticbackup](https://fosstodon.org/@restic) or by subscribing to the [project blog](https://restic.net/blog/). License diff --git a/VERSION b/VERSION index 4312e0d0c..201a22c8f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.15.2 +0.16.2 diff --git a/changelog/0.10.0_2020-09-19/pull-2668 b/changelog/0.10.0_2020-09-19/pull-2668 index 94a661c05..dd95587ce 100644 --- a/changelog/0.10.0_2020-09-19/pull-2668 +++ b/changelog/0.10.0_2020-09-19/pull-2668 @@ -1,6 +1,6 @@ Bugfix: Don't abort the stats command when data blobs are missing -Runing the stats command in the blobs-per-file mode on a repository with +Running the stats command in the blobs-per-file mode on a repository with missing data blobs previously resulted in a crash. https://github.com/restic/restic/pull/2668 diff --git a/changelog/0.11.0_2020-11-05/issue-1756 b/changelog/0.11.0_2020-11-05/issue-1756 index f735cf1f9..c182c1a6c 100644 --- a/changelog/0.11.0_2020-11-05/issue-1756 +++ b/changelog/0.11.0_2020-11-05/issue-1756 @@ -1,6 +1,6 @@ Bugfix: Mark repository files as read-only when using the local backend -Files stored in a local repository were marked as writeable on the +Files stored in a local repository were marked as writable on the filesystem for non-Windows systems, which did not prevent accidental file modifications outside of restic. In addition, the local backend did not work with certain filesystems and network mounts which do not permit modifications diff --git a/changelog/0.11.0_2020-11-05/issue-340 b/changelog/0.11.0_2020-11-05/issue-340 index 84c67f145..d688ee0db 100644 --- a/changelog/0.11.0_2020-11-05/issue-340 +++ b/changelog/0.11.0_2020-11-05/issue-340 @@ -5,7 +5,7 @@ another process using an exclusive lock through a filesystem snapshot. Restic was unable to backup those files before. This update enables backing up these files. -This needs to be enabled explicitely using the --use-fs-snapshot option of the +This needs to be enabled explicitly using the --use-fs-snapshot option of the backup command. https://github.com/restic/restic/issues/340 diff --git a/changelog/0.12.0_2021-02-14/pull-3106 b/changelog/0.12.0_2021-02-14/pull-3106 index 2d5857de7..f0cb54df0 100644 --- a/changelog/0.12.0_2021-02-14/pull-3106 +++ b/changelog/0.12.0_2021-02-14/pull-3106 @@ -2,7 +2,7 @@ Enhancement: Parallelize scan of snapshot content in `copy` and `prune` The `copy` and `prune` commands used to traverse the directories of snapshots one by one to find used data. This snapshot traversal is -now parallized which can speed up this step several times. +now parallelized which can speed up this step several times. In addition the `check` command now reports how many snapshots have already been processed. diff --git a/changelog/0.14.0_2022-08-25/pull-3475 b/changelog/0.14.0_2022-08-25/pull-3475 index 9932ae632..e79432faf 100644 --- a/changelog/0.14.0_2022-08-25/pull-3475 +++ b/changelog/0.14.0_2022-08-25/pull-3475 @@ -3,7 +3,7 @@ Enhancement: Allow limiting IO concurrency for local and SFTP backend Restic did not support limiting the IO concurrency / number of connections for accessing repositories stored using the local or SFTP backends. The number of connections is now limited as for other backends, and can be configured via the -the `-o local.connections=2` and `-o sftp.connections=5` options. This ensures -that restic does not overwhelm the backend with concurrent IO operations. +`-o local.connections=2` and `-o sftp.connections=5` options. This ensures that +restic does not overwhelm the backend with concurrent IO operations. https://github.com/restic/restic/pull/3475 diff --git a/changelog/0.16.0_2023-07-31/issue-1495 b/changelog/0.16.0_2023-07-31/issue-1495 new file mode 100644 index 000000000..b29f0d711 --- /dev/null +++ b/changelog/0.16.0_2023-07-31/issue-1495 @@ -0,0 +1,7 @@ +Enhancement: Sort snapshots by timestamp in `restic find` + +The `find` command used to print snapshots in an arbitrary order. Restic now +prints snapshots sorted by timestamp. + +https://github.com/restic/restic/issues/1495 +https://github.com/restic/restic/pull/4409 diff --git a/changelog/unreleased/issue-1759 b/changelog/0.16.0_2023-07-31/issue-1759 similarity index 60% rename from changelog/unreleased/issue-1759 rename to changelog/0.16.0_2023-07-31/issue-1759 index 1b698f845..6717dfe66 100644 --- a/changelog/unreleased/issue-1759 +++ b/changelog/0.16.0_2023-07-31/issue-1759 @@ -4,12 +4,13 @@ The `rebuild-index` command has been renamed to `repair index`. The old name will still work, but is deprecated. When a snapshot was damaged, the only option up to now was to completely forget -the snapshot, even if only some unimportant file was damaged. +the snapshot, even if only some unimportant files in it were damaged and other +files were still fine. -We've added a `repair snapshots` command, which can repair snapshots by removing -damaged directories and missing files contents. Note that using this command -can lead to data loss! Please see the "Troubleshooting" section in the documentation -for more details. +Restic now has a `repair snapshots` command, which can salvage any non-damaged +files and parts of files in the snapshots by removing damaged directories and +missing file contents. Please note that the damaged data may still be lost +and see the "Troubleshooting" section in the documentation for more details. https://github.com/restic/restic/issues/1759 https://github.com/restic/restic/issues/1714 diff --git a/changelog/0.16.0_2023-07-31/issue-1926 b/changelog/0.16.0_2023-07-31/issue-1926 new file mode 100644 index 000000000..96bc16240 --- /dev/null +++ b/changelog/0.16.0_2023-07-31/issue-1926 @@ -0,0 +1,8 @@ +Enhancement: Allow certificate paths to be passed through environment variables + +Restic will now read paths to certificates from the environment variables +`RESTIC_CACERT` or `RESTIC_TLS_CLIENT_CERT` if `--cacert` or `--tls-client-cert` +are not specified. + +https://github.com/restic/restic/issues/1926 +https://github.com/restic/restic/pull/4384 diff --git a/changelog/unreleased/issue-2359 b/changelog/0.16.0_2023-07-31/issue-2359 similarity index 52% rename from changelog/unreleased/issue-2359 rename to changelog/0.16.0_2023-07-31/issue-2359 index 0399a96f1..9a62aedfb 100644 --- a/changelog/unreleased/issue-2359 +++ b/changelog/0.16.0_2023-07-31/issue-2359 @@ -1,9 +1,9 @@ -Enhancement: Provide multi-platform Docker containers +Enhancement: Provide multi-platform Docker images -The official Docker containers are now built for the architectures linux/386, +The official Docker images are now built for the architectures linux/386, linux/amd64, linux/arm and linux/arm64. -As an alternative to the Docker Hub, the Docker containers are now also +As an alternative to the Docker Hub, the Docker images are also available on ghcr.io, the GitHub Container Registry. https://github.com/restic/restic/issues/2359 diff --git a/changelog/unreleased/issue-2468 b/changelog/0.16.0_2023-07-31/issue-2468 similarity index 61% rename from changelog/unreleased/issue-2468 rename to changelog/0.16.0_2023-07-31/issue-2468 index 56555a136..512c5e76a 100644 --- a/changelog/unreleased/issue-2468 +++ b/changelog/0.16.0_2023-07-31/issue-2468 @@ -1,10 +1,10 @@ Enhancement: Add support for non-global Azure clouds -Restic backups on Azure only supported storages using the global domain +The `azure` backend previously only supported storages using the global domain `core.windows.net`. This meant that backups to other domains such as Azure -China (`core.chinacloudapi.cn') or Azure Germany (`core.cloudapi.de`) were +China (`core.chinacloudapi.cn`) or Azure Germany (`core.cloudapi.de`) were not supported. Restic now allows overriding the global domain using the -environment variable `AZURE_ENDPOINT_SUFFIX'. +environment variable `AZURE_ENDPOINT_SUFFIX`. https://github.com/restic/restic/issues/2468 https://github.com/restic/restic/pull/4387 diff --git a/changelog/0.16.0_2023-07-31/issue-2565 b/changelog/0.16.0_2023-07-31/issue-2565 new file mode 100644 index 000000000..d7d5a25eb --- /dev/null +++ b/changelog/0.16.0_2023-07-31/issue-2565 @@ -0,0 +1,10 @@ +Bugfix: Support "unlimited" in `forget --keep-*` options + +Restic would previously forget snapshots that should have been kept when a +negative value was passed to the `--keep-*` options. Negative values are now +forbidden. To keep all snapshots, the special value `unlimited` is now +supported. For example, `--keep-monthly unlimited` will keep all monthly +snapshots. + +https://github.com/restic/restic/issues/2565 +https://github.com/restic/restic/pull/4234 diff --git a/changelog/0.16.0_2023-07-31/issue-3311 b/changelog/0.16.0_2023-07-31/issue-3311 new file mode 100644 index 000000000..af619b157 --- /dev/null +++ b/changelog/0.16.0_2023-07-31/issue-3311 @@ -0,0 +1,12 @@ +Bugfix: Support non-UTF8 paths as symlink target + +Earlier restic versions did not correctly `backup` and `restore` symlinks that +contain a non-UTF8 target. Note that this only affected systems that still use +a non-Unicode encoding for filesystem paths. + +The repository format is now extended to add support for such symlinks. Please +note that snapshots must have been created with at least restic version 0.16.0 +for `restore` to correctly handle non-UTF8 symlink targets when restoring them. + +https://github.com/restic/restic/issues/3311 +https://github.com/restic/restic/pull/3802 diff --git a/changelog/unreleased/issue-3328 b/changelog/0.16.0_2023-07-31/issue-3328 similarity index 52% rename from changelog/unreleased/issue-3328 rename to changelog/0.16.0_2023-07-31/issue-3328 index a8ef76d79..a79a4818b 100644 --- a/changelog/unreleased/issue-3328 +++ b/changelog/0.16.0_2023-07-31/issue-3328 @@ -1,5 +1,9 @@ Enhancement: Reduce memory usage by up to 25% +The in-memory index has been optimized to be more garbage collection friendly. +Restic now defaults to `GOGC=50` to run the Go garbage collector more +frequently. + https://github.com/restic/restic/issues/3328 https://github.com/restic/restic/pull/4352 https://github.com/restic/restic/pull/4353 diff --git a/changelog/0.16.0_2023-07-31/issue-3397 b/changelog/0.16.0_2023-07-31/issue-3397 new file mode 100644 index 000000000..31c5e19fb --- /dev/null +++ b/changelog/0.16.0_2023-07-31/issue-3397 @@ -0,0 +1,11 @@ +Enhancement: Improve accuracy of ETA displayed during backup + +Restic's `backup` command displayed an ETA that did not adapt when the rate of +progress made during the backup changed during the course of the backup. + +Restic now uses recent progress when computing the ETA. It is important to +realize that the estimate may still be wrong, because restic cannot predict +the future, but the hope is that the ETA will be more accurate in most cases. + +https://github.com/restic/restic/issues/3397 +https://github.com/restic/restic/pull/3563 diff --git a/changelog/unreleased/issue-3624 b/changelog/0.16.0_2023-07-31/issue-3624 similarity index 69% rename from changelog/unreleased/issue-3624 rename to changelog/0.16.0_2023-07-31/issue-3624 index ce3fe57aa..a2a11d2d4 100644 --- a/changelog/unreleased/issue-3624 +++ b/changelog/0.16.0_2023-07-31/issue-3624 @@ -1,8 +1,8 @@ Enhancement: Keep oldest snapshot when there are not enough snapshots The `forget` command now additionally preserves the oldest snapshot if fewer -snapshots are kept than allowed by the `--keep-*` parameters. This maximizes -amount of history kept while the specified limits are not yet reached. +snapshots than allowed by the `--keep-*` parameters would otherwise be kept. +This maximizes the amount of history kept within the specified limits. https://github.com/restic/restic/issues/3624 https://github.com/restic/restic/pull/4366 diff --git a/changelog/unreleased/issue-3698 b/changelog/0.16.0_2023-07-31/issue-3698 similarity index 58% rename from changelog/unreleased/issue-3698 rename to changelog/0.16.0_2023-07-31/issue-3698 index 0851d3756..95fc6bd69 100644 --- a/changelog/unreleased/issue-3698 +++ b/changelog/0.16.0_2023-07-31/issue-3698 @@ -1,7 +1,7 @@ -Enhancement: Add support for Managed / Worload Identity to azure backend +Enhancement: Add support for Managed / Workload Identity to `azure` backend Restic now additionally supports authenticating to Azure using Workload -Identity or Managed Identity credentials which are automatically injected in +Identity or Managed Identity credentials, which are automatically injected in several environments such as a managed Kubernetes cluster. https://github.com/restic/restic/issues/3698 diff --git a/changelog/0.16.0_2023-07-31/issue-3871 b/changelog/0.16.0_2023-07-31/issue-3871 new file mode 100644 index 000000000..45131bc19 --- /dev/null +++ b/changelog/0.16.0_2023-07-31/issue-3871 @@ -0,0 +1,22 @@ +Enhancement: Support `:` syntax to select subfolders + +Commands like `diff` or `restore` always worked with the full snapshot. This +did not allow comparing only a specific subfolder or only restoring that folder +(`restore --include subfolder` filters the restored files, but still creates the +directories included in `subfolder`). + +The commands `diff`, `dump`, `ls` and `restore` now support the +`:` syntax, where `snapshot` is the ID of a snapshot (or +the string `latest`) and `subfolder` is a path within the snapshot. The +commands will then only work with the specified path of the snapshot. The +`subfolder` must be a path to a folder as returned by `ls`. Two examples: + +`restic restore -t target latest:/some/path` +`restic diff 12345678:/some/path 90abcef:/some/path` + +For debugging purposes, the `cat` command now supports `cat tree +:` to return the directory metadata for the given +subfolder. + +https://github.com/restic/restic/issues/3871 +https://github.com/restic/restic/pull/4334 diff --git a/changelog/0.16.0_2023-07-31/issue-3941 b/changelog/0.16.0_2023-07-31/issue-3941 new file mode 100644 index 000000000..ff56d52cc --- /dev/null +++ b/changelog/0.16.0_2023-07-31/issue-3941 @@ -0,0 +1,17 @@ +Enhancement: Support `--group-by` for backup parent selection + +Previously, the `backup` command by default selected the parent snapshot based +on the hostname and the backup targets. When the backup path list changed, the +`backup` command was unable to determine a suitable parent snapshot and had to +read all files again. + +The new `--group-by` option for the `backup` command allows filtering snapshots +for the parent selection by `host`, `paths` and `tags`. It defaults to +`host,paths` which selects the latest snapshot with hostname and paths matching +those of the backup run. This matches the behavior of prior restic versions. + +The new `--group-by` option should be set to the same value as passed to +`forget --group-by`. + +https://github.com/restic/restic/issues/3941 +https://github.com/restic/restic/pull/4081 diff --git a/changelog/0.16.0_2023-07-31/issue-4130 b/changelog/0.16.0_2023-07-31/issue-4130 new file mode 100644 index 000000000..eeebf2c62 --- /dev/null +++ b/changelog/0.16.0_2023-07-31/issue-4130 @@ -0,0 +1,9 @@ +Enhancement: Cancel current command if cache becomes unusable + +If the cache directory was removed or ran out of space while restic was +running, this would previously cause further caching attempts to fail and +thereby drastically slow down the command execution. Now, the currently running +command is instead canceled. + +https://github.com/restic/restic/issues/4130 +https://github.com/restic/restic/pull/4166 diff --git a/changelog/0.16.0_2023-07-31/issue-4159 b/changelog/0.16.0_2023-07-31/issue-4159 new file mode 100644 index 000000000..4ef2fa846 --- /dev/null +++ b/changelog/0.16.0_2023-07-31/issue-4159 @@ -0,0 +1,12 @@ +Enhancement: Add `--human-readable` option to `ls` and `find` commands + +Previously, when using the `-l` option with the `ls` and `find` commands, the +displayed size was always in bytes, without an option for a more human readable +format such as MiB or GiB. + +The new `--human-readable` option will convert longer size values into more +human friendly values with an appropriate suffix depending on the output size. +For example, a size of `14680064` will be shown as `14.000 MiB`. + +https://github.com/restic/restic/issues/4159 +https://github.com/restic/restic/pull/4351 diff --git a/changelog/0.16.0_2023-07-31/issue-4188 b/changelog/0.16.0_2023-07-31/issue-4188 new file mode 100644 index 000000000..9bd5e6aca --- /dev/null +++ b/changelog/0.16.0_2023-07-31/issue-4188 @@ -0,0 +1,8 @@ +Enhancement: Include restic version in snapshot metadata + +The restic version used to backup a snapshot is now included in its metadata +and shown when inspecting a snapshot using `restic cat snapshot ` +or `restic snapshots --json`. + +https://github.com/restic/restic/issues/4188 +https://github.com/restic/restic/pull/4378 diff --git a/changelog/0.16.0_2023-07-31/issue-4199 b/changelog/0.16.0_2023-07-31/issue-4199 new file mode 100644 index 000000000..407fa43c6 --- /dev/null +++ b/changelog/0.16.0_2023-07-31/issue-4199 @@ -0,0 +1,9 @@ +Bugfix: Avoid lock refresh issues on slow network connections + +On network connections with a low upload speed, backups and other operations +could fail with the error message `Fatal: failed to refresh lock in time`. + +This has now been fixed by reworking the lock refresh handling. + +https://github.com/restic/restic/issues/4199 +https://github.com/restic/restic/pull/4304 diff --git a/changelog/unreleased/issue-426 b/changelog/0.16.0_2023-07-31/issue-426 similarity index 85% rename from changelog/unreleased/issue-426 rename to changelog/0.16.0_2023-07-31/issue-426 index 9caf14ef5..f50cddbf7 100644 --- a/changelog/unreleased/issue-426 +++ b/changelog/0.16.0_2023-07-31/issue-426 @@ -2,7 +2,7 @@ Enhancement: Show progress bar during restore The `restore` command now shows a progress report while restoring files. -Example: [0:42] 5.76% 23 files 12.98 MiB, total 3456 files 23.54 GiB +Example: `[0:42] 5.76% 23 files 12.98 MiB, total 3456 files 23.54 GiB` JSON output is now also supported. diff --git a/changelog/0.16.0_2023-07-31/issue-4274 b/changelog/0.16.0_2023-07-31/issue-4274 new file mode 100644 index 000000000..01c5f2177 --- /dev/null +++ b/changelog/0.16.0_2023-07-31/issue-4274 @@ -0,0 +1,11 @@ +Bugfix: Improve lock refresh handling after standby + +If the restic process was stopped or the host running restic entered standby +during a long running operation such as a backup, this previously resulted in +the operation failing with `Fatal: failed to refresh lock in time`. + +This has now been fixed such that restic first checks whether it is safe to +continue the current operation and only throws an error if not. + +https://github.com/restic/restic/issues/4274 +https://github.com/restic/restic/pull/4374 diff --git a/changelog/unreleased/issue-4375 b/changelog/0.16.0_2023-07-31/issue-4375 similarity index 100% rename from changelog/unreleased/issue-4375 rename to changelog/0.16.0_2023-07-31/issue-4375 diff --git a/changelog/0.16.0_2023-07-31/issue-719 b/changelog/0.16.0_2023-07-31/issue-719 new file mode 100644 index 000000000..897a6bf1f --- /dev/null +++ b/changelog/0.16.0_2023-07-31/issue-719 @@ -0,0 +1,8 @@ +Enhancement: Add `--retry-lock` option + +This option allows specifying a duration for which restic will wait if the +repository is already locked. + +https://github.com/restic/restic/issues/719 +https://github.com/restic/restic/pull/2214 +https://github.com/restic/restic/pull/4107 diff --git a/changelog/unreleased/pull-3261 b/changelog/0.16.0_2023-07-31/pull-3261 similarity index 100% rename from changelog/unreleased/pull-3261 rename to changelog/0.16.0_2023-07-31/pull-3261 diff --git a/changelog/unreleased/pull-4176 b/changelog/0.16.0_2023-07-31/pull-4176 similarity index 100% rename from changelog/unreleased/pull-4176 rename to changelog/0.16.0_2023-07-31/pull-4176 diff --git a/changelog/unreleased/pull-4201 b/changelog/0.16.0_2023-07-31/pull-4201 similarity index 78% rename from changelog/unreleased/pull-4201 rename to changelog/0.16.0_2023-07-31/pull-4201 index 500bbdbb1..8e4135f39 100644 --- a/changelog/unreleased/pull-4201 +++ b/changelog/0.16.0_2023-07-31/pull-4201 @@ -2,8 +2,6 @@ Change: Require Go 1.20 for Solaris builds Building restic on Solaris now requires Go 1.20, as the library used to access Azure uses the mmap syscall, which is only available on Solaris starting from -Go 1.20. - -All other platforms continue to build with Go 1.18. +Go 1.20. All other platforms however continue to build with Go 1.18. https://github.com/restic/restic/pull/4201 diff --git a/changelog/0.16.0_2023-07-31/pull-4220 b/changelog/0.16.0_2023-07-31/pull-4220 new file mode 100644 index 000000000..e832adf49 --- /dev/null +++ b/changelog/0.16.0_2023-07-31/pull-4220 @@ -0,0 +1,6 @@ +Enhancement: Add `jq` binary to Docker image + +The Docker image now contains `jq`, which can be useful to process JSON data +output by restic. + +https://github.com/restic/restic/pull/4220 diff --git a/changelog/0.16.0_2023-07-31/pull-4226 b/changelog/0.16.0_2023-07-31/pull-4226 new file mode 100644 index 000000000..1e04b35fb --- /dev/null +++ b/changelog/0.16.0_2023-07-31/pull-4226 @@ -0,0 +1,7 @@ +Enhancement: Allow specifying region of new buckets in the `gs` backend + +Previously, buckets used by the Google Cloud Storage backend would always get +created in the "us" region. It is now possible to specify the region where a +bucket should be created by using the `-o gs.region=us` option. + +https://github.com/restic/restic/pull/4226 diff --git a/changelog/unreleased/pull-4318 b/changelog/0.16.0_2023-07-31/pull-4318 similarity index 82% rename from changelog/unreleased/pull-4318 rename to changelog/0.16.0_2023-07-31/pull-4318 index 198b972d3..f13525f96 100644 --- a/changelog/unreleased/pull-4318 +++ b/changelog/0.16.0_2023-07-31/pull-4318 @@ -2,7 +2,7 @@ Bugfix: Correctly clean up status bar output of the `backup` command Due to a regression in restic 0.15.2, the status bar of the `backup` command could leave some output behind. This happened if filenames were printed that -are wider than the current terminal width. This has been fixed. +are wider than the current terminal width. This has now been fixed. https://github.com/restic/restic/issues/4319 https://github.com/restic/restic/pull/4318 diff --git a/changelog/unreleased/pull-4333 b/changelog/0.16.0_2023-07-31/pull-4333 similarity index 100% rename from changelog/unreleased/pull-4333 rename to changelog/0.16.0_2023-07-31/pull-4333 diff --git a/changelog/0.16.0_2023-07-31/pull-4400 b/changelog/0.16.0_2023-07-31/pull-4400 new file mode 100644 index 000000000..bce09842f --- /dev/null +++ b/changelog/0.16.0_2023-07-31/pull-4400 @@ -0,0 +1,8 @@ +Bugfix: Ignore missing folders in `rest` backend + +If a repository accessed via the REST backend was missing folders, then restic +would fail with an error while trying to list the data in the repository. This +has been now fixed. + +https://github.com/restic/restic/pull/4400 +https://github.com/restic/rest-server/issues/235 diff --git a/changelog/0.16.1_2023-10-24/issue-4128 b/changelog/0.16.1_2023-10-24/issue-4128 new file mode 100644 index 000000000..ac894656c --- /dev/null +++ b/changelog/0.16.1_2023-10-24/issue-4128 @@ -0,0 +1,9 @@ +Enhancement: Automatically set `GOMAXPROCS` in resource-constrained containers + +When running restic in a Linux container with CPU-usage limits, restic now +automatically adjusts `GOMAXPROCS`. This helps to reduce the memory consumption +on hosts with many CPU cores. + +https://github.com/restic/restic/issues/4128 +https://github.com/restic/restic/pull/4485 +https://github.com/restic/restic/pull/4531 diff --git a/changelog/0.16.1_2023-10-24/issue-4513 b/changelog/0.16.1_2023-10-24/issue-4513 new file mode 100644 index 000000000..7a26f34a4 --- /dev/null +++ b/changelog/0.16.1_2023-10-24/issue-4513 @@ -0,0 +1,8 @@ +Bugfix: Make `key list` command honor `--no-lock` + +The `key list` command now supports the `--no-lock` options. This allows +determining which keys a repo can be accessed by without the need for having +write access (e.g., read-only sftp access, filesystem snapshot). + +https://github.com/restic/restic/issues/4513 +https://github.com/restic/restic/pull/4514 diff --git a/changelog/0.16.1_2023-10-24/issue-4516 b/changelog/0.16.1_2023-10-24/issue-4516 new file mode 100644 index 000000000..de7833e33 --- /dev/null +++ b/changelog/0.16.1_2023-10-24/issue-4516 @@ -0,0 +1,8 @@ +Bugfix: Do not try to load password on command line autocomplete + +The command line autocompletion previously tried to load the repository +password. This could cause the autocompletion not to work. Now, this step gets +skipped. + +https://github.com/restic/restic/issues/4516 +https://github.com/restic/restic/pull/4526 diff --git a/changelog/0.16.1_2023-10-24/issue-4523 b/changelog/0.16.1_2023-10-24/issue-4523 new file mode 100644 index 000000000..e246cfe28 --- /dev/null +++ b/changelog/0.16.1_2023-10-24/issue-4523 @@ -0,0 +1,22 @@ +Bugfix: Update zstd library to fix possible data corruption at max. compression + +In restic 0.16.0, backups where the compression level was set to `max` (using +`--compression max`) could in rare and very specific circumstances result in +data corruption due to a bug in the library used for compressing data. + +Restic now uses the latest version of the library used to compress data, which +includes a fix for this issue. Please note that the `auto` compression level +(which restic uses by default) was never affected, and even if you used `max` +compression, chances of being affected by this issue were very small. + +To check a repository for any corruption, run `restic check --read-data`. This +will download and verify the whole repository and can be used at any time to +completely verify the integrity of a repository. If the `check` command detects +anomalies, follow the suggested steps. + +To simplify any needed repository repair and minimize data loss, there is also +a new and experimental `repair packs` command that salvages all valid data from +the affected pack files (see `restic help repair packs` for more information). + +https://github.com/restic/restic/issues/4523 +https://github.com/restic/restic/pull/4530 diff --git a/changelog/0.16.1_2023-10-24/pull-299 b/changelog/0.16.1_2023-10-24/pull-299 new file mode 100644 index 000000000..6774f3b9e --- /dev/null +++ b/changelog/0.16.1_2023-10-24/pull-299 @@ -0,0 +1,7 @@ +Enhancement: Show progress bar while loading the index + +Restic did not provide any feedback while loading index files. Now, there is a +progress bar that shows the index loading progress. + +https://github.com/restic/restic/issues/229 +https://github.com/restic/restic/pull/4419 diff --git a/changelog/0.16.1_2023-10-24/pull-4480 b/changelog/0.16.1_2023-10-24/pull-4480 new file mode 100644 index 000000000..2075fe21d --- /dev/null +++ b/changelog/0.16.1_2023-10-24/pull-4480 @@ -0,0 +1,11 @@ +Enhancement: Allow setting REST password and username via environment variables + +Previously, it was only possible to specify the REST-server username and +password in the repository URL, or by using the `--repository-file` option. +This meant it was not possible to use authentication in contexts where the +repository URL is stored in publicly accessible way. + +Restic now allows setting the username and password using the +`RESTIC_REST_USERNAME` and `RESTIC_REST_PASSWORD` variables. + +https://github.com/restic/restic/pull/4480 diff --git a/changelog/0.16.1_2023-10-24/pull-4511 b/changelog/0.16.1_2023-10-24/pull-4511 new file mode 100644 index 000000000..5bf81bebb --- /dev/null +++ b/changelog/0.16.1_2023-10-24/pull-4511 @@ -0,0 +1,7 @@ +Enhancement: Include inode numbers in JSON output for `find` and `ls` commands + +Restic used to omit the inode numbers in the JSON messages emitted for nodes by +the `ls` command as well as for matches by the `find` command. It now includes +those values whenever they are available. + +https://github.com/restic/restic/pull/4511 diff --git a/changelog/0.16.1_2023-10-24/pull-4519 b/changelog/0.16.1_2023-10-24/pull-4519 new file mode 100644 index 000000000..abc6445c2 --- /dev/null +++ b/changelog/0.16.1_2023-10-24/pull-4519 @@ -0,0 +1,12 @@ +Enhancement: Add config option to set SFTP command arguments + +When using the `sftp` backend, scenarios where a custom identity file was +needed for the SSH connection, required the full command to be specified: +`-o sftp.command='ssh user@host:port -i /ssh/my_private_key -s sftp'` + +Now, the `-o sftp.args=...` option can be passed to restic to specify +custom arguments for the SSH command executed by the SFTP backend. +This simplifies the above example to `-o sftp.args='-i /ssh/my_private_key'`. + +https://github.com/restic/restic/pull/4519 +https://github.com/restic/restic/issues/4241 diff --git a/changelog/0.16.1_2023-10-24/pull-4532 b/changelog/0.16.1_2023-10-24/pull-4532 new file mode 100644 index 000000000..4b6a06f34 --- /dev/null +++ b/changelog/0.16.1_2023-10-24/pull-4532 @@ -0,0 +1,8 @@ +Change: Update dependencies and require Go 1.19 or newer + +We have updated all dependencies. Since some libraries require newer Go +standard library features, support for Go 1.18 has been dropped, which means +that restic now requires at least Go 1.19 to build. + +https://github.com/restic/restic/pull/4532 +https://github.com/restic/restic/pull/4533 diff --git a/changelog/0.16.2_2023-10-29/issue-4540 b/changelog/0.16.2_2023-10-29/issue-4540 new file mode 100644 index 000000000..666bad59f --- /dev/null +++ b/changelog/0.16.2_2023-10-29/issue-4540 @@ -0,0 +1,9 @@ +Bugfix: Restore ARMv5 support for ARM binaries + +The official release binaries for restic 0.16.1 were accidentally built to +require ARMv7. The build process is now updated to restore support for ARMv5. + +Please note that restic 0.17.0 will drop support for ARMv5 and require at least +ARMv6. + +https://github.com/restic/restic/issues/4540 diff --git a/changelog/0.16.2_2023-10-29/pull-4545 b/changelog/0.16.2_2023-10-29/pull-4545 new file mode 100644 index 000000000..fd510a1e2 --- /dev/null +++ b/changelog/0.16.2_2023-10-29/pull-4545 @@ -0,0 +1,8 @@ +Bugfix: Repair documentation build on Read the Docs + +For restic 0.16.1, no documentation was available at +https://restic.readthedocs.io/ . + +The documentation build process is now updated to work again. + +https://github.com/restic/restic/pull/4545 diff --git a/changelog/0.8.0_2017-11-26/pull-1040 b/changelog/0.8.0_2017-11-26/pull-1040 index b39ee2fee..190ed01f8 100644 --- a/changelog/0.8.0_2017-11-26/pull-1040 +++ b/changelog/0.8.0_2017-11-26/pull-1040 @@ -3,7 +3,7 @@ Enhancement: Add local metadata cache We've added a local cache for metadata so that restic doesn't need to load all metadata (snapshots, indexes, ...) from the repo each time it starts. By default the cache is active, but there's a new global option `--no-cache` -that can be used to disable the cache. By deafult, the cache a standard +that can be used to disable the cache. By default, the cache a standard cache folder for the OS, which can be overridden with `--cache-dir`. The cache will automatically populate, indexes and snapshots are saved as they are loaded. Cache directories for repos that haven't been used recently can diff --git a/changelog/0.8.0_2017-11-26/pull-1319 b/changelog/0.8.0_2017-11-26/pull-1319 index d74a3f947..efc2e2c8a 100644 --- a/changelog/0.8.0_2017-11-26/pull-1319 +++ b/changelog/0.8.0_2017-11-26/pull-1319 @@ -1,6 +1,6 @@ Enhancement: Make `check` print `no errors found` explicitly -The `check` command now explicetly prints `No errors were found` when no errors +The `check` command now explicitly prints `No errors were found` when no errors could be found. https://github.com/restic/restic/pull/1319 diff --git a/changelog/0.8.2_2018-02-17/issue-1506 b/changelog/0.8.2_2018-02-17/issue-1506 index 5f0122529..aca77c458 100644 --- a/changelog/0.8.2_2018-02-17/issue-1506 +++ b/changelog/0.8.2_2018-02-17/issue-1506 @@ -1,4 +1,4 @@ -Bugfix: Limit bandwith at the http.RoundTripper for HTTP based backends +Bugfix: Limit bandwidth at the http.RoundTripper for HTTP based backends https://github.com/restic/restic/issues/1506 https://github.com/restic/restic/pull/1511 diff --git a/changelog/0.8.2_2018-02-17/pull-1595 b/changelog/0.8.2_2018-02-17/pull-1595 index 81e0a8748..3dbea73ce 100644 --- a/changelog/0.8.2_2018-02-17/pull-1595 +++ b/changelog/0.8.2_2018-02-17/pull-1595 @@ -1,7 +1,7 @@ Bugfix: backup: Remove bandwidth display This commit removes the bandwidth displayed during backup process. It is -misleading and seldomly correct, because it's neither the "read +misleading and seldom correct, because it's neither the "read bandwidth" (only for the very first backup) nor the "upload bandwidth". Many users are confused about (and rightly so), c.f. #1581, #1033, #1591 diff --git a/changelog/0.8.3_2018-02-26/pull-1623 b/changelog/0.8.3_2018-02-26/pull-1623 index 0e03ee776..1579a9ebc 100644 --- a/changelog/0.8.3_2018-02-26/pull-1623 +++ b/changelog/0.8.3_2018-02-26/pull-1623 @@ -6,7 +6,7 @@ that means making a request (e.g. via HTTP) and returning an error when the file already exists. This is not accurate, the file could have been created between the HTTP request -testing for it, and when writing starts, so we've relaxed this requeriment, +testing for it, and when writing starts, so we've relaxed this requirement, which saves one additional HTTP request per newly added file. https://github.com/restic/restic/pull/1623 diff --git a/changelog/0.9.0_2018-05-21/pull-1735 b/changelog/0.9.0_2018-05-21/pull-1735 index 2cfd115d8..fbf6135a6 100644 --- a/changelog/0.9.0_2018-05-21/pull-1735 +++ b/changelog/0.9.0_2018-05-21/pull-1735 @@ -1,4 +1,4 @@ -Enhancement: Allow keeping a time range of snaphots +Enhancement: Allow keeping a time range of snapshots We've added the `--keep-within` option to the `forget` command. It instructs restic to keep all snapshots within the given duration since the newest diff --git a/changelog/0.9.3_2018-10-13/pull-1876 b/changelog/0.9.3_2018-10-13/pull-1876 index 2fb1a8ea8..aa92b24e8 100644 --- a/changelog/0.9.3_2018-10-13/pull-1876 +++ b/changelog/0.9.3_2018-10-13/pull-1876 @@ -1,7 +1,7 @@ Enhancement: Display reason why forget keeps snapshots We've added a column to the list of snapshots `forget` keeps which details the -reasons to keep a particuliar snapshot. This makes debugging policies for +reasons to keep a particular snapshot. This makes debugging policies for forget much easier. Please remember to always try things out with `--dry-run`! https://github.com/restic/restic/pull/1876 diff --git a/changelog/0.9.6_2019-11-22/issue-2179 b/changelog/0.9.6_2019-11-22/issue-2179 index e87778d17..96589f9cf 100644 --- a/changelog/0.9.6_2019-11-22/issue-2179 +++ b/changelog/0.9.6_2019-11-22/issue-2179 @@ -9,7 +9,7 @@ file should be noticed, and the modified file will be backed up. The ctime check will be disabled if the --ignore-inode flag was given. If this change causes problems for you, please open an issue, and we can look in -to adding a seperate flag to disable just the ctime check. +to adding a separate flag to disable just the ctime check. https://github.com/restic/restic/issues/2179 https://github.com/restic/restic/pull/2212 diff --git a/changelog/CHANGELOG.tmpl b/changelog/CHANGELOG.tmpl index 712e7cc54..20f6254e7 100644 --- a/changelog/CHANGELOG.tmpl +++ b/changelog/CHANGELOG.tmpl @@ -1,18 +1,21 @@ -{{- range $changes := . }}{{ with $changes -}} -Changelog for restic {{ .Version }} ({{ .Date }}) -======================================= +# Table of Contents +{{ range . -}} + * [Changelog for {{ .Version }}](#changelog-for-restic-{{ .Version | replace "." ""}}-{{ .Date | lower -}}) +{{ end -}} + +{{- range $changes := . }}{{ with $changes }} + +# Changelog for restic {{ .Version }} ({{ .Date }}) The following sections list the changes in restic {{ .Version }} relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary {{ range $entry := .Entries }}{{ with $entry }} * {{ .TypeShort }} #{{ .PrimaryID }}: {{ .Title }} {{- end }}{{ end }} -Details -------- +## Details {{ range $entry := .Entries }}{{ with $entry }} * {{ .Type }} #{{ .PrimaryID }}: {{ .Title }} {{ range $par := .Paragraphs }} @@ -27,6 +30,5 @@ Details {{ range $url := .OtherURLs }} {{ $url -}} {{ end }} -{{ end }}{{ end }} - +{{ end }}{{ end -}} {{ end }}{{ end -}} diff --git a/changelog/TEMPLATE b/changelog/TEMPLATE index d512a2dc3..9304359b3 100644 --- a/changelog/TEMPLATE +++ b/changelog/TEMPLATE @@ -1,16 +1,17 @@ # The first line must start with Bugfix:, Enhancement: or Change:, -# including the colon. Use present tense. Remove lines starting with '#' -# from this template. +# including the colon. Use present tense and the imperative mood. Remove +# lines starting with '#' from this template. Enhancement: Allow custom bar in the foo command # Describe the problem in the past tense, the new behavior in the present # tense. Mention the affected commands, backends, operating systems, etc. # Focus on user-facing behavior, not the implementation. +# Use "Restic now ..." instead of "We have changed ...". Restic foo always used the system-wide bar when deciding how to frob an -item in the baz backend. It now permits selecting the bar with --bar or -the environment variable RESTIC_BAR. The system-wide bar is still the -default. +item in the `baz` backend. It now permits selecting the bar with `--bar` +or the environment variable `RESTIC_BAR`. The system-wide bar is still +the default. # The last section is a list of issue, PR and forum URLs. # The first issue ID determines the filename for the changelog entry: diff --git a/changelog/unreleased/issue-1926 b/changelog/unreleased/issue-1926 deleted file mode 100644 index 9f172b1f8..000000000 --- a/changelog/unreleased/issue-1926 +++ /dev/null @@ -1,8 +0,0 @@ -Enhancement: Certificates can be passed through environment variables - -Restic will now read the paths to the certificates from the environment -variables `RESTIC_CACERT` or `RESTIC_TLS_CLIENT_CERT` if `--cacert` or -`--tls-client-cert` are not specified. - -https://github.com/restic/restic/issues/1926 -https://github.com/restic/restic/pull/4384 diff --git a/changelog/unreleased/issue-2565 b/changelog/unreleased/issue-2565 deleted file mode 100644 index 4150dcda4..000000000 --- a/changelog/unreleased/issue-2565 +++ /dev/null @@ -1,8 +0,0 @@ -Bugfix: Restic forget --keep-* options now interpret "-1" as "forever" - -Restic would forget snapshots that should have been kept when "-1" was -used as a value for --keep-* options. It now interprets "-1" as forever, -e.g. an option like --keep-monthly -1 will keep all monthly snapshots. - -https://github.com/restic/restic/issues/2565 -https://github.com/restic/restic/pull/4234 diff --git a/changelog/unreleased/issue-3397 b/changelog/unreleased/issue-3397 deleted file mode 100644 index 391eeb004..000000000 --- a/changelog/unreleased/issue-3397 +++ /dev/null @@ -1,11 +0,0 @@ -Enhancement: Improve the ETA displayed during backup - -Restic's `backup` command displayed an ETA that did not adapt when the rate -of progress made during the backup changed during the course of the -backup. Restic now uses recent progress when computing the ETA. It is -important to realize that the estimate may still be wrong, because restic -cannot predict the future, but the hope is that the ETA will be more -accurate in most cases. - -https://github.com/restic/restic/issues/3397 -https://github.com/restic/restic/pull/3563 diff --git a/changelog/unreleased/issue-3941 b/changelog/unreleased/issue-3941 deleted file mode 100644 index 011cd9eaa..000000000 --- a/changelog/unreleased/issue-3941 +++ /dev/null @@ -1,14 +0,0 @@ -Enhancement: Support `--group-by` for backup parent selection - -The backup command by default selected the parent snapshot based on the hostname -and the backup targets. When the backup path list changed, the backup command -was unable to determine a suitable parent snapshot and had to read all -files again. - -The new `--group-by` option for the backup command allows filtering snapshots -for the parent selection by `host`, `paths` and `tags`. It defaults to -`host,paths` which selects the latest snapshot with hostname and paths matching -those of the backup run. It should be used consistently with `forget --group-by`. - -https://github.com/restic/restic/issues/3941 -https://github.com/restic/restic/pull/4081 diff --git a/changelog/unreleased/issue-4188 b/changelog/unreleased/issue-4188 deleted file mode 100644 index dbb26f733..000000000 --- a/changelog/unreleased/issue-4188 +++ /dev/null @@ -1,8 +0,0 @@ -Enhancement: `backup` includes restic version in snapshot metadata - -The restic version used backup the snapshot is now included in its metadata. -The program version is shown when inspecting a snapshot using `restic cat -snapshot ` or `restic snapshots --json`. - -https://github.com/restic/restic/issues/4188 -https://github.com/restic/restic/pull/4378 diff --git a/changelog/unreleased/issue-4251 b/changelog/unreleased/issue-4251 new file mode 100644 index 000000000..31be52401 --- /dev/null +++ b/changelog/unreleased/issue-4251 @@ -0,0 +1,14 @@ +Enhancement: Support reading backup from a program's standard output + +When reading data from stdin, the `backup` command could not verify whether the +corresponding command completed successfully. + +The `backup` command now supports starting an arbitrary command and sourcing +the backup content from its standard output. This enables restic to verify that +the command completes with exit code zero. A non-zero exit code causes the +backup to fail. + +Example: `restic backup --stdin-from-command mysqldump [...]` + +https://github.com/restic/restic/issues/4251 +https://github.com/restic/restic/pull/4410 diff --git a/changelog/unreleased/issue-4515 b/changelog/unreleased/issue-4515 new file mode 100644 index 000000000..3832dc605 --- /dev/null +++ b/changelog/unreleased/issue-4515 @@ -0,0 +1,8 @@ +Change: Don't retry to load files that don't exist + +Restic used to always retry to load files. It now only retries to load +files if they exist. + +https://github.com/restic/restic/issues/4515 +https://github.com/restic/restic/issues/1523 +https://github.com/restic/restic/pull/4520 diff --git a/changelog/unreleased/issue-4540 b/changelog/unreleased/issue-4540 new file mode 100644 index 000000000..9a706141e --- /dev/null +++ b/changelog/unreleased/issue-4540 @@ -0,0 +1,6 @@ +Change: Require at least ARMv6 for ARM binaries + +The official release binaries of restic now require at least ARMv6 support for ARM platforms. + +https://github.com/restic/restic/issues/4540 +https://github.com/restic/restic/pull/4542 diff --git a/changelog/unreleased/issue-4547 b/changelog/unreleased/issue-4547 new file mode 100644 index 000000000..edb1cf693 --- /dev/null +++ b/changelog/unreleased/issue-4547 @@ -0,0 +1,7 @@ +Enhancement: Add support for `--json` option to `version` command + +Restic now supports outputting restic version and used go version, OS and +architecture via JSON when using the version command. + +https://github.com/restic/restic/issues/4547 +https://github.com/restic/restic/pull/4553 diff --git a/changelog/unreleased/issue-719 b/changelog/unreleased/issue-719 deleted file mode 100644 index 4f28ea83c..000000000 --- a/changelog/unreleased/issue-719 +++ /dev/null @@ -1,8 +0,0 @@ -Enhancement: Add --retry-lock option - -This option allows to specify a duration for which restic will wait if there -already exists a conflicting lock within the repository. - -https://github.com/restic/restic/issues/719 -https://github.com/restic/restic/pull/2214 -https://github.com/restic/restic/pull/4107 diff --git a/changelog/unreleased/pull-4166 b/changelog/unreleased/pull-4166 deleted file mode 100644 index 6714fdf7f..000000000 --- a/changelog/unreleased/pull-4166 +++ /dev/null @@ -1,7 +0,0 @@ -Enhancement: Cancel current command if cache becomes unusable - -If the cache directory was removed or ran out of space while restic was -running, this caused further caching attempts to fail and drastically slow down -the command execution. Now, the currently running command is canceled instead. - -https://github.com/restic/restic/pull/4166 diff --git a/changelog/unreleased/pull-4220 b/changelog/unreleased/pull-4220 deleted file mode 100644 index 787b6ba2d..000000000 --- a/changelog/unreleased/pull-4220 +++ /dev/null @@ -1,5 +0,0 @@ -Enhancement: Add jq to container image - -The Docker container image now contains jq which can be useful when restic outputs json data. - -https://github.com/restic/restic/pull/4220 diff --git a/changelog/unreleased/pull-4226 b/changelog/unreleased/pull-4226 deleted file mode 100644 index 69c848735..000000000 --- a/changelog/unreleased/pull-4226 +++ /dev/null @@ -1,7 +0,0 @@ -Enhancement: Allow specifying the region of new buckets in the gcs backend - -Buckets used by the Google Cloud Storage backend would always get created in -the "us" region. It is now possible to specify the region, where a bucket -should get created. - -https://github.com/restic/restic/pull/4226 diff --git a/changelog/unreleased/pull-4304 b/changelog/unreleased/pull-4304 deleted file mode 100644 index ca3c7a8db..000000000 --- a/changelog/unreleased/pull-4304 +++ /dev/null @@ -1,5 +0,0 @@ -Bugfix: Avoid lock refresh issues with slow network connections - -On network connections with a low upload speed, restic could often fail backups and other operations with `Fatal: failed to refresh lock in time`. We've reworked the lock refresh to avoid this error. - -https://github.com/restic/restic/pull/4304 diff --git a/changelog/unreleased/pull-4365 b/changelog/unreleased/pull-4365 deleted file mode 100644 index c13a80af4..000000000 --- a/changelog/unreleased/pull-4365 +++ /dev/null @@ -1,6 +0,0 @@ -Change: Building restic on AIX is temporarily unsupported - -As the current version of the library used for the Azure backend does not -compile on AIX, there are currently no restic builds available for AIX. - -https://github.com/restic/restic/pull/4365 diff --git a/changelog/unreleased/pull-4503 b/changelog/unreleased/pull-4503 new file mode 100644 index 000000000..3ce5c48e8 --- /dev/null +++ b/changelog/unreleased/pull-4503 @@ -0,0 +1,7 @@ +Bugfix: Correct hardlink handling in `stats` command + +If files on different devices had the same inode id, then the `stats` command +did not correctly calculate the snapshot size. This has been fixed. + +https://github.com/restic/restic/pull/4503 +https://forum.restic.net/t/possible-bug-in-stats/6461/8 diff --git a/changelog/unreleased/pull-4573 b/changelog/unreleased/pull-4573 new file mode 100644 index 000000000..bd5c2c423 --- /dev/null +++ b/changelog/unreleased/pull-4573 @@ -0,0 +1,5 @@ +Enhancement: Add `--new-host` and `--new-time` options to `rewrite` command + +`restic rewrite` now allows rewriting the host and / or time metadata of a snapshot. + +https://github.com/restic/restic/pull/4573 diff --git a/changelog/unreleased/pull-4590 b/changelog/unreleased/pull-4590 new file mode 100644 index 000000000..353d21616 --- /dev/null +++ b/changelog/unreleased/pull-4590 @@ -0,0 +1,7 @@ +Enhancement: `mount` tests mountpoint existence before opening the repository + +The restic `mount` command now checks for the existence of the +mountpoint before opening the repository, leading to quicker error +detection. + +https://github.com/restic/restic/pull/4590 diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index d7e899eaf..a2b81a759 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -97,6 +97,7 @@ type BackupOptions struct { ExcludeLargerThan string Stdin bool StdinFilename string + StdinCommand bool Tags restic.TagLists Host string FilesFrom []string @@ -134,6 +135,7 @@ func init() { f.StringVar(&backupOptions.ExcludeLargerThan, "exclude-larger-than", "", "max `size` of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)") f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin") f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin") + f.BoolVar(&backupOptions.StdinCommand, "stdin-from-command", false, "execute command and store its stdout") f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)") f.UintVar(&backupOptions.ReadConcurrency, "read-concurrency", 0, "read `n` files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)") f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag") @@ -287,7 +289,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error { } } - if opts.Stdin { + if opts.Stdin || opts.StdinCommand { if len(opts.FilesFrom) > 0 { return errors.Fatal("--stdin and --files-from cannot be used together") } @@ -298,7 +300,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error { return errors.Fatal("--stdin and --files-from-raw cannot be used together") } - if len(args) > 0 { + if len(args) > 0 && !opts.StdinCommand { return errors.Fatal("--stdin was specified and files/dirs were listed as arguments") } } @@ -366,7 +368,7 @@ func collectRejectFuncs(opts BackupOptions, targets []string) (fs []RejectFunc, // collectTargets returns a list of target files/dirs from several sources. func collectTargets(opts BackupOptions, args []string) (targets []string, err error) { - if opts.Stdin { + if opts.Stdin || opts.StdinCommand { return nil, nil } @@ -453,7 +455,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup f.Tags = []restic.TagList{opts.Tags.Flatten()} } - sn, err := f.FindLatest(ctx, repo.Backend(), repo, snName) + sn, _, err := f.FindLatest(ctx, repo, repo, snName) // Snapshot not found is ok if no explicit parent was set if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) { err = nil @@ -506,10 +508,13 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter if !gopts.JSON { progressPrinter.V("lock repository") } - lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err + if !opts.DryRun { + var lock *restic.Lock + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) + defer unlockRepo(lock) + if err != nil { + return err + } } // rejectByNameFuncs collect functions that can reject items from the backup based on path only @@ -543,7 +548,10 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter if !gopts.JSON { progressPrinter.V("load index files") } - err = repo.LoadIndex(ctx) + + bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term) + + err = repo.LoadIndex(ctx, bar) if err != nil { return err } @@ -586,16 +594,24 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter defer localVss.DeleteSnapshots() targetFS = localVss } - if opts.Stdin { + + if opts.Stdin || opts.StdinCommand { if !gopts.JSON { progressPrinter.V("read data from stdin") } filename := path.Join("/", opts.StdinFilename) + var source io.ReadCloser = os.Stdin + if opts.StdinCommand { + source, err = fs.NewCommandReader(ctx, args, globalOptions.stderr) + if err != nil { + return err + } + } targetFS = &fs.Reader{ ModTime: timeStamp, Name: filename, Mode: 0644, - ReadCloser: os.Stdin, + ReadCloser: source, } targets = []string{filename} } @@ -624,7 +640,13 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter success := true arch.Error = func(item string, err error) error { success = false - return progressReporter.Error(item, err) + reterr := progressReporter.Error(item, err) + // If we receive a fatal error during the execution of the snapshot, + // we abort the snapshot. + if reterr == nil && errors.IsFatal(err) { + reterr = err + } + return reterr } arch.CompleteItem = progressReporter.CompleteItem arch.StartFile = progressReporter.StartFile diff --git a/cmd/restic/cmd_backup_integration_test.go b/cmd/restic/cmd_backup_integration_test.go index fb7bef633..c60e9c543 100644 --- a/cmd/restic/cmd_backup_integration_test.go +++ b/cmd/restic/cmd_backup_integration_test.go @@ -9,6 +9,7 @@ import ( "runtime" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -252,7 +253,7 @@ func TestBackupTreeLoadError(t *testing.T) { r, err := OpenRepository(context.TODO(), env.gopts) rtest.OK(t, err) - rtest.OK(t, r.LoadIndex(context.TODO())) + rtest.OK(t, r.LoadIndex(context.TODO(), nil)) treePacks := restic.NewIDSet() r.Index().Each(context.TODO(), func(pb restic.PackedBlob) { if pb.Type == restic.TreeBlob { @@ -265,7 +266,7 @@ func TestBackupTreeLoadError(t *testing.T) { // delete the subdirectory pack first for id := range treePacks { - rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()})) + rtest.OK(t, r.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()})) } testRunRebuildIndex(t, env.gopts) // now the repo is missing the tree blob in the index; check should report this @@ -567,3 +568,72 @@ func linkEqual(source, dest []string) bool { return true } + +func TestStdinFromCommand(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{ + StdinCommand: true, + StdinFilename: "stdin", + } + + testRunBackup(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; print('something'); sys.exit(0)"}, opts, env.gopts) + testListSnapshots(t, env.gopts, 1) + + testRunCheck(t, env.gopts) +} + +func TestStdinFromCommandNoOutput(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{ + StdinCommand: true, + StdinFilename: "stdin", + } + + err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; sys.exit(0)"}, opts, env.gopts) + rtest.Assert(t, err != nil && err.Error() == "at least one source file could not be read", "No data error expected") + testListSnapshots(t, env.gopts, 1) + + testRunCheck(t, env.gopts) +} + +func TestStdinFromCommandFailExitCode(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{ + StdinCommand: true, + StdinFilename: "stdin", + } + + err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; print('test'); sys.exit(1)"}, opts, env.gopts) + rtest.Assert(t, err != nil, "Expected error while backing up") + + testListSnapshots(t, env.gopts, 0) + + testRunCheck(t, env.gopts) +} + +func TestStdinFromCommandFailNoOutputAndExitCode(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{ + StdinCommand: true, + StdinFilename: "stdin", + } + + err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; sys.exit(1)"}, opts, env.gopts) + rtest.Assert(t, err != nil, "Expected error while backing up") + + testListSnapshots(t, env.gopts, 0) + + testRunCheck(t, env.gopts) +} diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index 771731a58..92f58b2e7 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "strings" "github.com/spf13/cobra" @@ -13,7 +14,7 @@ import ( ) var cmdCat = &cobra.Command{ - Use: "cat [flags] [pack|blob|snapshot|index|key|masterkey|config|lock] ID", + Use: "cat [flags] [masterkey|config|pack ID|blob ID|snapshot ID|index ID|key ID|lock ID|tree snapshot:subfolder]", Short: "Print internal objects to stdout", Long: ` The "cat" command is used to print internal objects to stdout. @@ -33,9 +34,34 @@ func init() { cmdRoot.AddCommand(cmdCat) } +func validateCatArgs(args []string) error { + var allowedCmds = []string{"config", "index", "snapshot", "key", "masterkey", "lock", "pack", "blob", "tree"} + + if len(args) < 1 { + return errors.Fatal("type not specified") + } + + validType := false + for _, v := range allowedCmds { + if v == args[0] { + validType = true + break + } + } + if !validType { + return errors.Fatalf("invalid type %q, must be one of [%s]", args[0], strings.Join(allowedCmds, "|")) + } + + if args[0] != "masterkey" && args[0] != "config" && len(args) != 2 { + return errors.Fatal("ID not specified") + } + + return nil +} + func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { - if len(args) < 1 || (args[0] != "masterkey" && args[0] != "config" && len(args) != 2) { - return errors.Fatal("type or ID not specified") + if err := validateCatArgs(args); err != nil { + return err } repo, err := OpenRepository(ctx, gopts) @@ -55,7 +81,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { tpe := args[0] var id restic.ID - if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" { + if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" && tpe != "tree" { id, err = restic.ParseID(args[1]) if err != nil { return errors.Fatalf("unable to parse ID: %v\n", err) @@ -80,7 +106,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { Println(string(buf)) return nil case "snapshot": - sn, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1]) + sn, _, err := restic.FindSnapshot(ctx, repo, repo, args[1]) if err != nil { return errors.Fatalf("could not find snapshot: %v\n", err) } @@ -128,7 +154,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { return nil case "pack": - h := restic.Handle{Type: restic.PackFile, Name: id.String()} + h := backend.Handle{Type: restic.PackFile, Name: id.String()} buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h) if err != nil { return err @@ -143,7 +169,8 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { return err case "blob": - err = repo.LoadIndex(ctx) + bar := newIndexProgress(gopts.Quiet, gopts.JSON) + err = repo.LoadIndex(ctx, bar) if err != nil { return err } @@ -165,6 +192,30 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { return errors.Fatal("blob not found") + case "tree": + sn, subfolder, err := restic.FindSnapshot(ctx, repo, repo, args[1]) + if err != nil { + return errors.Fatalf("could not find snapshot: %v\n", err) + } + + bar := newIndexProgress(gopts.Quiet, gopts.JSON) + err = repo.LoadIndex(ctx, bar) + if err != nil { + return err + } + + sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder) + if err != nil { + return err + } + + buf, err := repo.LoadBlob(ctx, restic.TreeBlob, *sn.Tree, nil) + if err != nil { + return err + } + _, err = globalOptions.stdout.Write(buf) + return err + default: return errors.Fatal("invalid type") } diff --git a/cmd/restic/cmd_cat_test.go b/cmd/restic/cmd_cat_test.go new file mode 100644 index 000000000..8c72a16a9 --- /dev/null +++ b/cmd/restic/cmd_cat_test.go @@ -0,0 +1,30 @@ +package main + +import ( + "strings" + "testing" + + rtest "github.com/restic/restic/internal/test" +) + +func TestCatArgsValidation(t *testing.T) { + for _, test := range []struct { + args []string + err string + }{ + {[]string{}, "Fatal: type not specified"}, + {[]string{"masterkey"}, ""}, + {[]string{"invalid"}, `Fatal: invalid type "invalid"`}, + {[]string{"snapshot"}, "Fatal: ID not specified"}, + {[]string{"snapshot", "12345678"}, ""}, + } { + t.Run("", func(t *testing.T) { + err := validateCatArgs(test.args) + if test.err == "" { + rtest.Assert(t, err == nil, "unexpected error %q", err) + } else { + rtest.Assert(t, strings.Contains(err.Error(), test.err), "unexpected error expected %q to contain %q", err, test.err) + } + }) + } +} diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index 3c4c9daa9..f04a4fe71 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -226,7 +226,8 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args } Verbosef("load indexes\n") - hints, errs := chkr.LoadIndex(ctx) + bar := newIndexProgress(gopts.Quiet, gopts.JSON) + hints, errs := chkr.LoadIndex(ctx, bar) errorsFound := false suggestIndexRebuild := false @@ -329,11 +330,28 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args go chkr.ReadPacks(ctx, packs, p, errChan) + var salvagePacks restic.IDs + for err := range errChan { errorsFound = true Warnf("%v\n", err) + if err, ok := err.(*checker.ErrPackData); ok { + if strings.Contains(err.Error(), "wrong data returned, hash is") { + salvagePacks = append(salvagePacks, err.PackID) + } + } } p.Done() + + if len(salvagePacks) > 0 { + Warnf("\nThe repository contains pack files with damaged blobs. These blobs must be removed to repair the repository. This can be done using the following commands:\n\n") + var strIds []string + for _, id := range salvagePacks { + strIds = append(strIds, id.String()) + } + Warnf("RESTIC_FEATURES=repair-packs-v1 restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIds, " ")) + Warnf("Corrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n") + } } switch { @@ -399,7 +417,7 @@ func selectPacksByBucket(allPacks map[restic.ID]int64, bucket, totalBuckets uint return packs } -// selectRandomPacksByPercentage selects the given percentage of packs which are randomly choosen. +// selectRandomPacksByPercentage selects the given percentage of packs which are randomly chosen. func selectRandomPacksByPercentage(allPacks map[restic.ID]int64, percentage float64) map[restic.ID]int64 { packCount := len(allPacks) packsToCheck := int(float64(packCount) * (percentage / 100.0)) diff --git a/cmd/restic/cmd_check_test.go b/cmd/restic/cmd_check_test.go index fb61f8420..4d54488cd 100644 --- a/cmd/restic/cmd_check_test.go +++ b/cmd/restic/cmd_check_test.go @@ -71,7 +71,7 @@ func TestSelectPacksByBucket(t *testing.T) { var testPacks = make(map[restic.ID]int64) for i := 1; i <= 10; i++ { id := restic.NewRandomID() - // ensure relevant part of generated id is reproducable + // ensure relevant part of generated id is reproducible id[0] = byte(i) testPacks[id] = 0 } @@ -124,7 +124,7 @@ func TestSelectRandomPacksByPercentage(t *testing.T) { } func TestSelectNoRandomPacksByPercentage(t *testing.T) { - // that the a repository without pack files works + // that the repository without pack files works var testPacks = make(map[restic.ID]int64) selectedPacks := selectRandomPacksByPercentage(testPacks, 10.0) rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs") @@ -158,7 +158,7 @@ func TestSelectRandomPacksByFileSize(t *testing.T) { } func TestSelectNoRandomPacksByFileSize(t *testing.T) { - // that the a repository without pack files works + // that the repository without pack files works var testPacks = make(map[restic.ID]int64) selectedPacks := selectRandomPacksByFileSize(testPacks, 10, 500) rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs") diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index eaa0ef81a..0cf96a092 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -4,7 +4,6 @@ import ( "context" "fmt" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" @@ -88,23 +87,24 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] return err } - srcSnapshotLister, err := backend.MemorizeList(ctx, srcRepo.Backend(), restic.SnapshotFile) + srcSnapshotLister, err := restic.MemorizeList(ctx, srcRepo, restic.SnapshotFile) if err != nil { return err } - dstSnapshotLister, err := backend.MemorizeList(ctx, dstRepo.Backend(), restic.SnapshotFile) + dstSnapshotLister, err := restic.MemorizeList(ctx, dstRepo, restic.SnapshotFile) if err != nil { return err } debug.Log("Loading source index") - if err := srcRepo.LoadIndex(ctx); err != nil { + bar := newIndexProgress(gopts.Quiet, gopts.JSON) + if err := srcRepo.LoadIndex(ctx, bar); err != nil { return err } - + bar = newIndexProgress(gopts.Quiet, gopts.JSON) debug.Log("Loading destination index") - if err := dstRepo.LoadIndex(ctx); err != nil { + if err := dstRepo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index a54200c45..60413de21 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -78,7 +78,7 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error { } func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error { - return restic.ForAllSnapshots(ctx, repo.Backend(), repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error { + return restic.ForAllSnapshots(ctx, repo, repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error { if err != nil { return err } @@ -107,7 +107,7 @@ type Blob struct { func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) error { var m sync.Mutex - return restic.ParallelList(ctx, repo.Backend(), restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error { + return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error { blobs, _, err := repo.ListPack(ctx, id, size) if err != nil { Warnf("error for pack %v: %v\n", id.Str(), err) @@ -134,7 +134,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) } func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error { - return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { + return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { Printf("index_id: %v\n", id) if err != nil { return err @@ -290,7 +290,7 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by }) err := wg.Wait() if err != nil { - panic("all go rountines can only return nil") + panic("all go routines can only return nil") } if !found { @@ -321,7 +321,7 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li panic(err) } be := repo.Backend() - h := restic.Handle{ + h := backend.Handle{ Name: packID.String(), Type: restic.PackFile, } @@ -447,7 +447,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er for _, name := range args { id, err := restic.ParseID(name) if err != nil { - id, err = restic.Find(ctx, repo.Backend(), restic.PackFile, name) + id, err = restic.Find(ctx, repo, restic.PackFile, name) if err != nil { Warnf("error: %v\n", err) continue @@ -469,7 +469,8 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er } } - err = repo.LoadIndex(ctx) + bar := newIndexProgress(gopts.Quiet, gopts.JSON) + err = repo.LoadIndex(ctx, bar) if err != nil { return err } @@ -489,7 +490,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) error { Printf("examine %v\n", id) - h := restic.Handle{ + h := backend.Handle{ Type: restic.PackFile, Name: id.String(), } diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index 3c59b9580..c54fc06d4 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -7,7 +7,6 @@ import ( "reflect" "sort" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -16,7 +15,7 @@ import ( ) var cmdDiff = &cobra.Command{ - Use: "diff [flags] snapshot-ID snapshot-ID", + Use: "diff [flags] snapshotID snapshotID", Short: "Show differences between two snapshots", Long: ` The "diff" command shows differences from the first to the second snapshot. The @@ -29,6 +28,10 @@ directory: * M The file's content was modified * T The type was changed, e.g. a file was made a symlink +To only compare files in specific subfolders, you can use the +":" syntax, where "subfolder" is a path within the +snapshot. + EXIT STATUS =========== @@ -54,12 +57,12 @@ func init() { f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata") } -func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, error) { - sn, err := restic.FindSnapshot(ctx, be, repo, desc) +func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) { + sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc) if err != nil { - return nil, errors.Fatal(err.Error()) + return nil, "", errors.Fatal(err.Error()) } - return sn, err + return sn, subfolder, err } // Comparer collects all things needed to compare two snapshots. @@ -342,16 +345,16 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args [] } // cache snapshots listing - be, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) + be, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile) if err != nil { return err } - sn1, err := loadSnapshot(ctx, be, repo, args[0]) + sn1, subfolder1, err := loadSnapshot(ctx, be, repo, args[0]) if err != nil { return err } - sn2, err := loadSnapshot(ctx, be, repo, args[1]) + sn2, subfolder2, err := loadSnapshot(ctx, be, repo, args[1]) if err != nil { return err } @@ -359,8 +362,8 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args [] if !gopts.JSON { Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str()) } - - if err = repo.LoadIndex(ctx); err != nil { + bar := newIndexProgress(gopts.Quiet, gopts.JSON) + if err = repo.LoadIndex(ctx, bar); err != nil { return err } @@ -372,6 +375,16 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args [] return errors.Errorf("snapshot %v has nil tree", sn2.ID().Str()) } + sn1.Tree, err = restic.FindTreeDirectory(ctx, repo, sn1.Tree, subfolder1) + if err != nil { + return err + } + + sn2.Tree, err = restic.FindTreeDirectory(ctx, repo, sn2.Tree, subfolder2) + if err != nil { + return err + } + c := &Comparer{ repo: repo, opts: diffOptions, diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index 34313f582..e6020d847 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -24,9 +24,13 @@ single file is selected, it prints its contents to stdout. Folders are output as a tar (default) or zip file containing the contents of the specified folder. Pass "/" as file name to dump the whole snapshot as an archive file. -The special snapshot "latest" can be used to use the latest snapshot in the +The special snapshotID "latest" can be used to use the latest snapshot in the repository. +To include the folder content at the root of the archive, you can use the +":" syntax, where "subfolder" is a path within the +snapshot. + EXIT STATUS =========== @@ -139,16 +143,22 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] } } - sn, err := (&restic.SnapshotFilter{ + sn, subfolder, err := (&restic.SnapshotFilter{ Hosts: opts.Hosts, Paths: opts.Paths, Tags: opts.Tags, - }).FindLatest(ctx, repo.Backend(), repo, snapshotIDString) + }).FindLatest(ctx, repo, repo, snapshotIDString) if err != nil { return errors.Fatalf("failed to find snapshot: %v", err) } - err = repo.LoadIndex(ctx) + bar := newIndexProgress(gopts.Quiet, gopts.JSON) + err = repo.LoadIndex(ctx, bar) + if err != nil { + return err + } + + sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder) if err != nil { return err } diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 6b5e32df7..c30650823 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -9,7 +9,6 @@ import ( "github.com/spf13/cobra" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/filter" @@ -51,6 +50,7 @@ type FindOptions struct { PackID, ShowPackID bool CaseInsensitive bool ListLong bool + HumanReadable bool restic.SnapshotFilter } @@ -69,6 +69,7 @@ func init() { f.BoolVar(&findOptions.ShowPackID, "show-pack-id", false, "display the pack-ID the blobs belong to (with --blob or --tree)") f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern") f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode") + f.BoolVar(&findOptions.HumanReadable, "human-readable", false, "print sizes in human readable format") initMultiSnapshotFilter(f, &findOptions.SnapshotFilter, true) } @@ -104,12 +105,13 @@ func parseTime(str string) (time.Time, error) { } type statefulOutput struct { - ListLong bool - JSON bool - inuse bool - newsn *restic.Snapshot - oldsn *restic.Snapshot - hits int + ListLong bool + HumanReadable bool + JSON bool + inuse bool + newsn *restic.Snapshot + oldsn *restic.Snapshot + hits int } func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) { @@ -123,7 +125,6 @@ func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) { // Make the following attributes disappear Name byte `json:"name,omitempty"` - Inode byte `json:"inode,omitempty"` ExtendedAttributes byte `json:"extended_attributes,omitempty"` Device byte `json:"device,omitempty"` Content byte `json:"content,omitempty"` @@ -164,7 +165,7 @@ func (s *statefulOutput) PrintPatternNormal(path string, node *restic.Node) { s.oldsn = s.newsn Verbosef("Found matching entries in snapshot %s from %s\n", s.oldsn.ID().Str(), s.oldsn.Time.Local().Format(TimeFormat)) } - Println(formatNode(path, node, s.ListLong)) + Println(formatNode(path, node, s.ListLong, s.HumanReadable)) } func (s *statefulOutput) PrintPattern(path string, node *restic.Node) { @@ -582,19 +583,19 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args [] } } - snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) + snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile) if err != nil { return err } - - if err = repo.LoadIndex(ctx); err != nil { + bar := newIndexProgress(gopts.Quiet, gopts.JSON) + if err = repo.LoadIndex(ctx, bar); err != nil { return err } f := &Finder{ repo: repo, pat: pat, - out: statefulOutput{ListLong: opts.ListLong, JSON: gopts.JSON}, + out: statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON}, ignoreTrees: restic.NewIDSet(), } @@ -618,7 +619,16 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args [] } } + var filteredSnapshots []*restic.Snapshot for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, opts.Snapshots) { + filteredSnapshots = append(filteredSnapshots, sn) + } + + sort.Slice(filteredSnapshots, func(i, j int) bool { + return filteredSnapshots[i].Time.Before(filteredSnapshots[j].Time) + }) + + for _, sn := range filteredSnapshots { if f.blobIDs != nil || f.treeIDs != nil { if err = f.findIDs(ctx, sn); err != nil && err.Error() != "OK" { return err diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index d8e64bc6a..a7f39dc4e 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "io" + "strconv" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -36,14 +37,49 @@ Exit status is 0 if the command was successful, and non-zero if there was any er }, } +type ForgetPolicyCount int + +var ErrNegativePolicyCount = errors.New("negative values not allowed, use 'unlimited' instead") + +func (c *ForgetPolicyCount) Set(s string) error { + switch s { + case "unlimited": + *c = -1 + default: + val, err := strconv.ParseInt(s, 10, 0) + if err != nil { + return err + } + if val < 0 { + return ErrNegativePolicyCount + } + *c = ForgetPolicyCount(val) + } + + return nil +} + +func (c *ForgetPolicyCount) String() string { + switch *c { + case -1: + return "unlimited" + default: + return strconv.FormatInt(int64(*c), 10) + } +} + +func (c *ForgetPolicyCount) Type() string { + return "n" +} + // ForgetOptions collects all options for the forget command. type ForgetOptions struct { - Last int - Hourly int - Daily int - Weekly int - Monthly int - Yearly int + Last ForgetPolicyCount + Hourly ForgetPolicyCount + Daily ForgetPolicyCount + Weekly ForgetPolicyCount + Monthly ForgetPolicyCount + Yearly ForgetPolicyCount Within restic.Duration WithinHourly restic.Duration WithinDaily restic.Duration @@ -67,12 +103,12 @@ func init() { cmdRoot.AddCommand(cmdForget) f := cmdForget.Flags() - f.IntVarP(&forgetOptions.Last, "keep-last", "l", 0, "keep the last `n` snapshots (use '-1' to keep all snapshots)") - f.IntVarP(&forgetOptions.Hourly, "keep-hourly", "H", 0, "keep the last `n` hourly snapshots (use '-1' to keep all hourly snapshots)") - f.IntVarP(&forgetOptions.Daily, "keep-daily", "d", 0, "keep the last `n` daily snapshots (use '-1' to keep all daily snapshots)") - f.IntVarP(&forgetOptions.Weekly, "keep-weekly", "w", 0, "keep the last `n` weekly snapshots (use '-1' to keep all weekly snapshots)") - f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots (use '-1' to keep all monthly snapshots)") - f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots (use '-1' to keep all yearly snapshots)") + f.VarP(&forgetOptions.Last, "keep-last", "l", "keep the last `n` snapshots (use 'unlimited' to keep all snapshots)") + f.VarP(&forgetOptions.Hourly, "keep-hourly", "H", "keep the last `n` hourly snapshots (use 'unlimited' to keep all hourly snapshots)") + f.VarP(&forgetOptions.Daily, "keep-daily", "d", "keep the last `n` daily snapshots (use 'unlimited' to keep all daily snapshots)") + f.VarP(&forgetOptions.Weekly, "keep-weekly", "w", "keep the last `n` weekly snapshots (use 'unlimited' to keep all weekly snapshots)") + f.VarP(&forgetOptions.Monthly, "keep-monthly", "m", "keep the last `n` monthly snapshots (use 'unlimited' to keep all monthly snapshots)") + f.VarP(&forgetOptions.Yearly, "keep-yearly", "y", "keep the last `n` yearly snapshots (use 'unlimited' to keep all yearly snapshots)") f.VarP(&forgetOptions.Within, "keep-within", "", "keep snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot") f.VarP(&forgetOptions.WithinHourly, "keep-within-hourly", "", "keep hourly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot") f.VarP(&forgetOptions.WithinDaily, "keep-within-daily", "", "keep daily snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot") @@ -147,7 +183,7 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg var snapshots restic.Snapshots removeSnIDs := restic.NewIDSet() - for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) { + for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) { snapshots = append(snapshots, sn) } @@ -165,12 +201,12 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg } policy := restic.ExpirePolicy{ - Last: opts.Last, - Hourly: opts.Hourly, - Daily: opts.Daily, - Weekly: opts.Weekly, - Monthly: opts.Monthly, - Yearly: opts.Yearly, + Last: int(opts.Last), + Hourly: int(opts.Hourly), + Daily: int(opts.Daily), + Weekly: int(opts.Weekly), + Monthly: int(opts.Monthly), + Yearly: int(opts.Yearly), Within: opts.Within, WithinHourly: opts.WithinHourly, WithinDaily: opts.WithinDaily, diff --git a/cmd/restic/cmd_forget_test.go b/cmd/restic/cmd_forget_test.go index 9fd5c7bb0..ddeef028a 100644 --- a/cmd/restic/cmd_forget_test.go +++ b/cmd/restic/cmd_forget_test.go @@ -7,55 +7,84 @@ import ( rtest "github.com/restic/restic/internal/test" ) +func TestForgetPolicyValues(t *testing.T) { + testCases := []struct { + input string + value ForgetPolicyCount + err string + }{ + {"0", ForgetPolicyCount(0), ""}, + {"1", ForgetPolicyCount(1), ""}, + {"unlimited", ForgetPolicyCount(-1), ""}, + {"", ForgetPolicyCount(0), "strconv.ParseInt: parsing \"\": invalid syntax"}, + {"-1", ForgetPolicyCount(0), ErrNegativePolicyCount.Error()}, + {"abc", ForgetPolicyCount(0), "strconv.ParseInt: parsing \"abc\": invalid syntax"}, + } + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + var count ForgetPolicyCount + err := count.Set(testCase.input) + + if testCase.err != "" { + rtest.Assert(t, err != nil, "should have returned error for input %+v", testCase.input) + rtest.Equals(t, testCase.err, err.Error()) + } else { + rtest.Assert(t, err == nil, "expected no error for input %+v, got %v", testCase.input, err) + rtest.Equals(t, testCase.value, count) + rtest.Equals(t, testCase.input, count.String()) + } + }) + } +} + func TestForgetOptionValues(t *testing.T) { const negValErrorMsg = "Fatal: negative values other than -1 are not allowed for --keep-*" const negDurationValErrorMsg = "Fatal: durations containing negative values are not allowed for --keep-within*" testCases := []struct { - input ForgetOptions - expectsError bool - errorMsg string + input ForgetOptions + errorMsg string }{ - {ForgetOptions{Last: 1}, false, ""}, - {ForgetOptions{Hourly: 1}, false, ""}, - {ForgetOptions{Daily: 1}, false, ""}, - {ForgetOptions{Weekly: 1}, false, ""}, - {ForgetOptions{Monthly: 1}, false, ""}, - {ForgetOptions{Yearly: 1}, false, ""}, - {ForgetOptions{Last: 0}, false, ""}, - {ForgetOptions{Hourly: 0}, false, ""}, - {ForgetOptions{Daily: 0}, false, ""}, - {ForgetOptions{Weekly: 0}, false, ""}, - {ForgetOptions{Monthly: 0}, false, ""}, - {ForgetOptions{Yearly: 0}, false, ""}, - {ForgetOptions{Last: -1}, false, ""}, - {ForgetOptions{Hourly: -1}, false, ""}, - {ForgetOptions{Daily: -1}, false, ""}, - {ForgetOptions{Weekly: -1}, false, ""}, - {ForgetOptions{Monthly: -1}, false, ""}, - {ForgetOptions{Yearly: -1}, false, ""}, - {ForgetOptions{Last: -2}, true, negValErrorMsg}, - {ForgetOptions{Hourly: -2}, true, negValErrorMsg}, - {ForgetOptions{Daily: -2}, true, negValErrorMsg}, - {ForgetOptions{Weekly: -2}, true, negValErrorMsg}, - {ForgetOptions{Monthly: -2}, true, negValErrorMsg}, - {ForgetOptions{Yearly: -2}, true, negValErrorMsg}, - {ForgetOptions{Within: restic.ParseDurationOrPanic("1y2m3d3h")}, false, ""}, - {ForgetOptions{WithinHourly: restic.ParseDurationOrPanic("1y2m3d3h")}, false, ""}, - {ForgetOptions{WithinDaily: restic.ParseDurationOrPanic("1y2m3d3h")}, false, ""}, - {ForgetOptions{WithinWeekly: restic.ParseDurationOrPanic("1y2m3d3h")}, false, ""}, - {ForgetOptions{WithinMonthly: restic.ParseDurationOrPanic("2y4m6d8h")}, false, ""}, - {ForgetOptions{WithinYearly: restic.ParseDurationOrPanic("2y4m6d8h")}, false, ""}, - {ForgetOptions{Within: restic.ParseDurationOrPanic("-1y2m3d3h")}, true, negDurationValErrorMsg}, - {ForgetOptions{WithinHourly: restic.ParseDurationOrPanic("1y-2m3d3h")}, true, negDurationValErrorMsg}, - {ForgetOptions{WithinDaily: restic.ParseDurationOrPanic("1y2m-3d3h")}, true, negDurationValErrorMsg}, - {ForgetOptions{WithinWeekly: restic.ParseDurationOrPanic("1y2m3d-3h")}, true, negDurationValErrorMsg}, - {ForgetOptions{WithinMonthly: restic.ParseDurationOrPanic("-2y4m6d8h")}, true, negDurationValErrorMsg}, - {ForgetOptions{WithinYearly: restic.ParseDurationOrPanic("2y-4m6d8h")}, true, negDurationValErrorMsg}, + {ForgetOptions{Last: 1}, ""}, + {ForgetOptions{Hourly: 1}, ""}, + {ForgetOptions{Daily: 1}, ""}, + {ForgetOptions{Weekly: 1}, ""}, + {ForgetOptions{Monthly: 1}, ""}, + {ForgetOptions{Yearly: 1}, ""}, + {ForgetOptions{Last: 0}, ""}, + {ForgetOptions{Hourly: 0}, ""}, + {ForgetOptions{Daily: 0}, ""}, + {ForgetOptions{Weekly: 0}, ""}, + {ForgetOptions{Monthly: 0}, ""}, + {ForgetOptions{Yearly: 0}, ""}, + {ForgetOptions{Last: -1}, ""}, + {ForgetOptions{Hourly: -1}, ""}, + {ForgetOptions{Daily: -1}, ""}, + {ForgetOptions{Weekly: -1}, ""}, + {ForgetOptions{Monthly: -1}, ""}, + {ForgetOptions{Yearly: -1}, ""}, + {ForgetOptions{Last: -2}, negValErrorMsg}, + {ForgetOptions{Hourly: -2}, negValErrorMsg}, + {ForgetOptions{Daily: -2}, negValErrorMsg}, + {ForgetOptions{Weekly: -2}, negValErrorMsg}, + {ForgetOptions{Monthly: -2}, negValErrorMsg}, + {ForgetOptions{Yearly: -2}, negValErrorMsg}, + {ForgetOptions{Within: restic.ParseDurationOrPanic("1y2m3d3h")}, ""}, + {ForgetOptions{WithinHourly: restic.ParseDurationOrPanic("1y2m3d3h")}, ""}, + {ForgetOptions{WithinDaily: restic.ParseDurationOrPanic("1y2m3d3h")}, ""}, + {ForgetOptions{WithinWeekly: restic.ParseDurationOrPanic("1y2m3d3h")}, ""}, + {ForgetOptions{WithinMonthly: restic.ParseDurationOrPanic("2y4m6d8h")}, ""}, + {ForgetOptions{WithinYearly: restic.ParseDurationOrPanic("2y4m6d8h")}, ""}, + {ForgetOptions{Within: restic.ParseDurationOrPanic("-1y2m3d3h")}, negDurationValErrorMsg}, + {ForgetOptions{WithinHourly: restic.ParseDurationOrPanic("1y-2m3d3h")}, negDurationValErrorMsg}, + {ForgetOptions{WithinDaily: restic.ParseDurationOrPanic("1y2m-3d3h")}, negDurationValErrorMsg}, + {ForgetOptions{WithinWeekly: restic.ParseDurationOrPanic("1y2m3d-3h")}, negDurationValErrorMsg}, + {ForgetOptions{WithinMonthly: restic.ParseDurationOrPanic("-2y4m6d8h")}, negDurationValErrorMsg}, + {ForgetOptions{WithinYearly: restic.ParseDurationOrPanic("2y-4m6d8h")}, negDurationValErrorMsg}, } for _, testCase := range testCases { err := verifyForgetOptions(&testCase.input) - if testCase.expectsError { + if testCase.errorMsg != "" { rtest.Assert(t, err != nil, "should have returned error for input %+v", testCase.input) rtest.Equals(t, testCase.errorMsg, err.Error()) } else { diff --git a/cmd/restic/cmd_init.go b/cmd/restic/cmd_init.go index b9dabdc2d..7154279e8 100644 --- a/cmd/restic/cmd_init.go +++ b/cmd/restic/cmd_init.go @@ -75,7 +75,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args [] return err } - repo, err := ReadRepo(gopts) + gopts.Repo, err = ReadRepo(gopts) if err != nil { return err } @@ -87,7 +87,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args [] return err } - be, err := create(ctx, repo, gopts, gopts.extended) + be, err := create(ctx, gopts.Repo, gopts, gopts.extended) if err != nil { return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.backends, gopts.Repo), err) } diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index 62521d762..e147f537e 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -7,6 +7,7 @@ import ( "strings" "sync" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -59,7 +60,7 @@ func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions var m sync.Mutex var keys []keyInfo - err := restic.ParallelList(ctx, s.Backend(), restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, size int64) error { + err := restic.ParallelList(ctx, s, restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, size int64) error { k, err := repository.LoadKey(ctx, s, id) if err != nil { Warnf("LoadKey() failed: %v\n", err) @@ -149,7 +150,7 @@ func deleteKey(ctx context.Context, repo *repository.Repository, id restic.ID) e return errors.Fatal("refusing to remove key currently used to access repository") } - h := restic.Handle{Type: restic.KeyFile, Name: id.String()} + h := backend.Handle{Type: restic.KeyFile, Name: id.String()} err := repo.Backend().Remove(ctx, h) if err != nil { return err @@ -176,7 +177,7 @@ func changePassword(ctx context.Context, repo *repository.Repository, gopts Glob return err } - h := restic.Handle{Type: restic.KeyFile, Name: oldID.String()} + h := backend.Handle{Type: restic.KeyFile, Name: oldID.String()} err = repo.Backend().Remove(ctx, h) if err != nil { return err @@ -193,7 +194,7 @@ func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repos err := repo.SearchKey(ctx, pw, 0, key.ID().String()) if err != nil { // the key is invalid, try to remove it - h := restic.Handle{Type: restic.KeyFile, Name: key.ID().String()} + h := backend.Handle{Type: restic.KeyFile, Name: key.ID().String()} _ = repo.Backend().Remove(ctx, h) return errors.Fatalf("failed to access repository with new key: %v", err) } @@ -212,10 +213,13 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error { switch args[0] { case "list": - lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err + if !gopts.NoLock { + var lock *restic.Lock + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) + defer unlockRepo(lock) + if err != nil { + return err + } } return listKeys(ctx, repo, gopts) @@ -234,7 +238,7 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error { return err } - id, err := restic.Find(ctx, repo.Backend(), restic.KeyFile, args[1]) + id, err := restic.Find(ctx, repo, restic.KeyFile, args[1]) if err != nil { return err } diff --git a/cmd/restic/cmd_key_integration_test.go b/cmd/restic/cmd_key_integration_test.go index 9ea5795ba..f68799dde 100644 --- a/cmd/restic/cmd_key_integration_test.go +++ b/cmd/restic/cmd_key_integration_test.go @@ -6,8 +6,8 @@ import ( "regexp" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -110,11 +110,11 @@ func TestKeyAddRemove(t *testing.T) { } type emptySaveBackend struct { - restic.Backend + backend.Backend } -func (b *emptySaveBackend) Save(ctx context.Context, h restic.Handle, _ restic.RewindReader) error { - return b.Backend.Save(ctx, h, restic.NewByteReader([]byte{}, nil)) +func (b *emptySaveBackend) Save(ctx context.Context, h backend.Handle, _ backend.RewindReader) error { + return b.Backend.Save(ctx, h, backend.NewByteReader([]byte{}, nil)) } func TestKeyProblems(t *testing.T) { @@ -122,7 +122,7 @@ func TestKeyProblems(t *testing.T) { defer cleanup() testRunInit(t, env.gopts) - env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return &emptySaveBackend{r}, nil } diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go index bd02cedc7..38f8b094a 100644 --- a/cmd/restic/cmd_list.go +++ b/cmd/restic/cmd_list.go @@ -63,7 +63,7 @@ func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args case "locks": t = restic.LockFile case "blobs": - return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { + return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { if err != nil { return err } diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index e8c27381c..5b3984eb2 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -9,7 +9,6 @@ import ( "github.com/spf13/cobra" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" @@ -50,7 +49,8 @@ Exit status is 0 if the command was successful, and non-zero if there was any er type LsOptions struct { ListLong bool restic.SnapshotFilter - Recursive bool + Recursive bool + HumanReadable bool } var lsOptions LsOptions @@ -62,6 +62,7 @@ func init() { initSingleSnapshotFilter(flags, &lsOptions.SnapshotFilter) flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode") flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories") + flags.BoolVar(&lsOptions.HumanReadable, "human-readable", false, "print sizes in human readable format") } type lsSnapshot struct { @@ -85,6 +86,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { ModTime time.Time `json:"mtime,omitempty"` AccessTime time.Time `json:"atime,omitempty"` ChangeTime time.Time `json:"ctime,omitempty"` + Inode uint64 `json:"inode,omitempty"` StructType string `json:"struct_type"` // "node" size uint64 // Target for Size pointer. @@ -100,6 +102,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { ModTime: node.ModTime, AccessTime: node.AccessTime, ChangeTime: node.ChangeTime, + Inode: node.Inode, StructType: "node", } // Always print size for regular files, even when empty, @@ -166,12 +169,13 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return err } - snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) + snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile) if err != nil { return err } - if err = repo.LoadIndex(ctx); err != nil { + bar := newIndexProgress(gopts.Quiet, gopts.JSON) + if err = repo.LoadIndex(ctx, bar); err != nil { return err } @@ -206,11 +210,11 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri Verbosef("snapshot %s of %v filtered by %v at %s):\n", sn.ID().Str(), sn.Paths, dirs, sn.Time) } printNode = func(path string, node *restic.Node) { - Printf("%s\n", formatNode(path, node, lsOptions.ListLong)) + Printf("%s\n", formatNode(path, node, lsOptions.ListLong, lsOptions.HumanReadable)) } } - sn, err := (&restic.SnapshotFilter{ + sn, subfolder, err := (&restic.SnapshotFilter{ Hosts: opts.Hosts, Paths: opts.Paths, Tags: opts.Tags, @@ -219,6 +223,11 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return err } + sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder) + if err != nil { + return err + } + printSnapshot(sn) err = walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index ec3662d5c..5fd81b344 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -113,6 +113,15 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args return errors.Fatal("wrong number of parameters") } + mountpoint := args[0] + + // Check the existence of the mount point at the earliest stage to + // prevent unnecessary computations while opening the repository. + if _, err := resticfs.Stat(mountpoint); errors.Is(err, os.ErrNotExist) { + Verbosef("Mountpoint %s doesn't exist\n", mountpoint) + return err + } + debug.Log("start mount") defer debug.Log("finish mount") @@ -130,17 +139,12 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args } } - err = repo.LoadIndex(ctx) + bar := newIndexProgress(gopts.Quiet, gopts.JSON) + err = repo.LoadIndex(ctx, bar) if err != nil { return err } - mountpoint := args[0] - - if _, err := resticfs.Stat(mountpoint); errors.Is(err, os.ErrNotExist) { - Verbosef("Mountpoint %s doesn't exist\n", mountpoint) - return err - } mountOptions := []systemFuse.MountOption{ systemFuse.ReadOnly(), systemFuse.FSName("restic"), diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index e4c2c7b29..739a450df 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -152,7 +152,7 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error return err } - if repo.Backend().Connections() < 2 { + if repo.Connections() < 2 { return errors.Fatal("prune requires a backend connection limit of at least two") } @@ -187,7 +187,8 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption Verbosef("loading indexes...\n") // loading the index before the snapshots is ok, as we use an exclusive lock here - err := repo.LoadIndex(ctx) + bar := newIndexProgress(gopts.Quiet, gopts.JSON) + err := repo.LoadIndex(ctx, bar) if err != nil { return err } @@ -405,7 +406,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re }) // if duplicate blobs exist, those will be set to either "used" or "unused": - // - mark only one occurence of duplicate blobs as used + // - mark only one occurrence of duplicate blobs as used // - if there are already some used blobs in a pack, possibly mark duplicates in this pack as "used" // - if there are no used blobs in a pack, possibly mark duplicates as "unused" if hasDuplicates { @@ -414,7 +415,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re bh := blob.BlobHandle count, ok := usedBlobs[bh] // skip non-duplicate, aka. normal blobs - // count == 0 is used to mark that this was a duplicate blob with only a single occurence remaining + // count == 0 is used to mark that this was a duplicate blob with only a single occurrence remaining if !ok || count == 1 { return } @@ -423,7 +424,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re size := uint64(blob.Length) switch { case ip.usedBlobs > 0, count == 0: - // other used blobs in pack or "last" occurence -> transition to used + // other used blobs in pack or "last" occurrence -> transition to used ip.usedSize += size ip.usedBlobs++ ip.unusedSize -= size @@ -433,7 +434,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re stats.blobs.used++ stats.size.duplicate -= size stats.blobs.duplicate-- - // let other occurences remain marked as unused + // let other occurrences remain marked as unused usedBlobs[bh] = 1 default: // remain unused and decrease counter @@ -809,7 +810,7 @@ func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Rep func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (usedBlobs restic.CountedBlobSet, err error) { var snapshotTrees restic.IDs Verbosef("loading all snapshots...\n") - err = restic.ForAllSnapshots(ctx, repo.Backend(), repo, ignoreSnapshots, + err = restic.ForAllSnapshots(ctx, repo, repo, ignoreSnapshots, func(id restic.ID, sn *restic.Snapshot, err error) error { if err != nil { debug.Log("failed to load snapshot %v (error %v)", id, err) diff --git a/cmd/restic/cmd_prune_integration_test.go b/cmd/restic/cmd_prune_integration_test.go index 2cd86d895..53e27ee10 100644 --- a/cmd/restic/cmd_prune_integration_test.go +++ b/cmd/restic/cmd_prune_integration_test.go @@ -6,13 +6,13 @@ import ( "path/filepath" "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" rtest "github.com/restic/restic/internal/test" ) func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) { oldHook := gopts.backendTestHook - gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil } + gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil } defer func() { gopts.backendTestHook = oldHook }() @@ -130,7 +130,7 @@ func TestPruneWithDamagedRepository(t *testing.T) { removePacksExcept(env.gopts, t, oldPacks, false) oldHook := env.gopts.backendTestHook - env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil } + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil } defer func() { env.gopts.backendTestHook = oldHook }() diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index 85dcc23d7..ae6aff740 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -5,7 +5,6 @@ import ( "os" "time" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" "github.com/spf13/cobra" @@ -52,13 +51,14 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error { return err } - snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) + snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile) if err != nil { return err } Verbosef("load index files\n") - if err = repo.LoadIndex(ctx); err != nil { + bar := newIndexProgress(gopts.Quiet, gopts.JSON) + if err = repo.LoadIndex(ctx, bar); err != nil { return err } @@ -73,7 +73,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error { }) Verbosef("load %d trees\n", len(trees)) - bar := newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded") + bar = newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded") for id := range trees { tree, err := restic.LoadTree(ctx, repo, id) if err != nil { diff --git a/cmd/restic/cmd_repair_index.go b/cmd/restic/cmd_repair_index.go index b1905836a..c8a94b470 100644 --- a/cmd/restic/cmd_repair_index.go +++ b/cmd/restic/cmd_repair_index.go @@ -88,7 +88,7 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti } else { Verbosef("loading indexes...\n") mi := index.NewMasterIndex() - err := index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { + err := index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { if err != nil { Warnf("removing invalid index %v: %v\n", id, err) obsoleteIndexes = append(obsoleteIndexes, id) diff --git a/cmd/restic/cmd_repair_index_integration_test.go b/cmd/restic/cmd_repair_index_integration_test.go index f451173a3..e3271361a 100644 --- a/cmd/restic/cmd_repair_index_integration_test.go +++ b/cmd/restic/cmd_repair_index_integration_test.go @@ -8,6 +8,7 @@ import ( "sync" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/restic" @@ -70,12 +71,12 @@ func TestRebuildIndexAlwaysFull(t *testing.T) { // indexErrorBackend modifies the first index after reading. type indexErrorBackend struct { - restic.Backend + backend.Backend lock sync.Mutex hasErred bool } -func (b *indexErrorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { +func (b *indexErrorBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error { // protect hasErred b.lock.Lock() @@ -101,7 +102,7 @@ func (erd errorReadCloser) Read(p []byte) (int, error) { } func TestRebuildIndexDamage(t *testing.T) { - testRebuildIndex(t, func(r restic.Backend) (restic.Backend, error) { + testRebuildIndex(t, func(r backend.Backend) (backend.Backend, error) { return &indexErrorBackend{ Backend: r, }, nil @@ -109,11 +110,11 @@ func TestRebuildIndexDamage(t *testing.T) { } type appendOnlyBackend struct { - restic.Backend + backend.Backend } // called via repo.Backend().Remove() -func (b *appendOnlyBackend) Remove(_ context.Context, h restic.Handle) error { +func (b *appendOnlyBackend) Remove(_ context.Context, h backend.Handle) error { return errors.Errorf("Failed to remove %v", h) } @@ -127,7 +128,7 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) { err := withRestoreGlobalOptions(func() error { globalOptions.stdout = io.Discard - env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return &appendOnlyBackend{r}, nil } return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts) diff --git a/cmd/restic/cmd_repair_packs.go b/cmd/restic/cmd_repair_packs.go new file mode 100644 index 000000000..7d1a3a392 --- /dev/null +++ b/cmd/restic/cmd_repair_packs.go @@ -0,0 +1,159 @@ +package main + +import ( + "context" + "io" + "os" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" +) + +var cmdRepairPacks = &cobra.Command{ + Use: "packs [packIDs...]", + Short: "Salvage damaged pack files", + Long: ` +WARNING: The CLI for this command is experimental and will likely change in the future! + +The "repair packs" command extracts intact blobs from the specified pack files, rebuilds +the index to remove the damaged pack files and removes the pack files from the repository. + +EXIT STATUS +=========== + +Exit status is 0 if the command was successful, and non-zero if there was any error. +`, + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + return runRepairPacks(cmd.Context(), globalOptions, args) + }, +} + +func init() { + cmdRepair.AddCommand(cmdRepairPacks) +} + +func runRepairPacks(ctx context.Context, gopts GlobalOptions, args []string) error { + // FIXME discuss and add proper feature flag mechanism + flag, _ := os.LookupEnv("RESTIC_FEATURES") + if flag != "repair-packs-v1" { + return errors.Fatal("This command is experimental and may change/be removed without notice between restic versions. " + + "Set the environment variable 'RESTIC_FEATURES=repair-packs-v1' to enable it.") + } + + ids := restic.NewIDSet() + for _, arg := range args { + id, err := restic.ParseID(arg) + if err != nil { + return err + } + ids.Insert(id) + } + if len(ids) == 0 { + return errors.Fatal("no ids specified") + } + + repo, err := OpenRepository(ctx, gopts) + if err != nil { + return err + } + + lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) + defer unlockRepo(lock) + if err != nil { + return err + } + + return repairPacks(ctx, gopts, repo, ids) +} + +func repairPacks(ctx context.Context, gopts GlobalOptions, repo *repository.Repository, ids restic.IDSet) error { + bar := newIndexProgress(gopts.Quiet, gopts.JSON) + err := repo.LoadIndex(ctx, bar) + if err != nil { + return errors.Fatalf("%s", err) + } + + Warnf("saving backup copies of pack files in current folder\n") + for id := range ids { + f, err := os.OpenFile("pack-"+id.String(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o666) + if err != nil { + return errors.Fatalf("%s", err) + } + + err = repo.Backend().Load(ctx, backend.Handle{Type: restic.PackFile, Name: id.String()}, 0, 0, func(rd io.Reader) error { + _, err := f.Seek(0, 0) + if err != nil { + return err + } + _, err = io.Copy(f, rd) + return err + }) + if err != nil { + return errors.Fatalf("%s", err) + } + } + + wg, wgCtx := errgroup.WithContext(ctx) + repo.StartPackUploader(wgCtx, wg) + repo.DisableAutoIndexUpdate() + + Warnf("salvaging intact data from specified pack files\n") + bar = newProgressMax(!gopts.Quiet, uint64(len(ids)), "pack files") + defer bar.Done() + + wg.Go(func() error { + // examine all data the indexes have for the pack file + for b := range repo.Index().ListPacks(wgCtx, ids) { + blobs := b.Blobs + if len(blobs) == 0 { + Warnf("no blobs found for pack %v\n", b.PackID) + bar.Add(1) + continue + } + + err = repository.StreamPack(wgCtx, repo.Backend().Load, repo.Key(), b.PackID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { + if err != nil { + // Fallback path + buf, err = repo.LoadBlob(wgCtx, blob.Type, blob.ID, nil) + if err != nil { + Warnf("failed to load blob %v: %v\n", blob.ID, err) + return nil + } + } + id, _, _, err := repo.SaveBlob(wgCtx, blob.Type, buf, restic.ID{}, true) + if !id.Equal(blob.ID) { + panic("pack id mismatch during upload") + } + return err + }) + if err != nil { + return err + } + bar.Add(1) + } + return repo.Flush(wgCtx) + }) + + if err := wg.Wait(); err != nil { + return errors.Fatalf("%s", err) + } + bar.Done() + + // remove salvaged packs from index + err = rebuildIndexFiles(ctx, gopts, repo, ids, nil) + if err != nil { + return errors.Fatalf("%s", err) + } + + // cleanup + Warnf("removing salvaged pack files\n") + DeleteFiles(ctx, gopts, repo, ids, restic.PackFile) + + Warnf("\nUse `restic repair snapshots --forget` to remove the corrupted data blobs from all snapshots\n") + return nil +} diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index 03736795c..786097132 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -3,7 +3,6 @@ package main import ( "context" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/walker" @@ -84,12 +83,13 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt repo.SetDryRun() } - snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) + snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile) if err != nil { return err } - if err := repo.LoadIndex(ctx); err != nil { + bar := newIndexProgress(gopts.Quiet, gopts.JSON) + if err := repo.LoadIndex(ctx, bar); err != nil { return err } @@ -148,7 +148,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt changed, err := filterAndReplaceSnapshot(ctx, repo, sn, func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) - }, opts.DryRun, opts.Forget, "repaired") + }, opts.DryRun, opts.Forget, nil, "repaired") if err != nil { return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err) } diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index c59ac34de..6045a5d41 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -25,9 +25,12 @@ var cmdRestore = &cobra.Command{ The "restore" command extracts the data from a snapshot from the repository to a directory. -The special snapshot "latest" can be used to restore the latest snapshot in the +The special snapshotID "latest" can be used to restore the latest snapshot in the repository. +To only restore a specific subfolder, you can use the ":" +syntax, where "subfolder" is a path within the snapshot. + EXIT STATUS =========== @@ -82,9 +85,9 @@ func init() { flags := cmdRestore.Flags() flags.StringArrayVarP(&restoreOptions.Exclude, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)") - flags.StringArrayVar(&restoreOptions.InsensitiveExclude, "iexclude", nil, "same as `--exclude` but ignores the casing of filenames") + flags.StringArrayVar(&restoreOptions.InsensitiveExclude, "iexclude", nil, "same as --exclude but ignores the casing of `pattern`") flags.StringArrayVarP(&restoreOptions.Include, "include", "i", nil, "include a `pattern`, exclude everything else (can be specified multiple times)") - flags.StringArrayVar(&restoreOptions.InsensitiveInclude, "iinclude", nil, "same as `--include` but ignores the casing of filenames") + flags.StringArrayVar(&restoreOptions.InsensitiveInclude, "iinclude", nil, "same as --include but ignores the casing of `pattern`") flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to") initSingleSnapshotFilter(flags, &restoreOptions.SnapshotFilter) @@ -161,16 +164,22 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, } } - sn, err := (&restic.SnapshotFilter{ + sn, subfolder, err := (&restic.SnapshotFilter{ Hosts: opts.Hosts, Paths: opts.Paths, Tags: opts.Tags, - }).FindLatest(ctx, repo.Backend(), repo, snapshotIDString) + }).FindLatest(ctx, repo, repo, snapshotIDString) if err != nil { return errors.Fatalf("failed to find snapshot: %v", err) } - err = repo.LoadIndex(ctx) + bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term) + err = repo.LoadIndex(ctx, bar) + if err != nil { + return err + } + + sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder) if err != nil { return err } diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index c08797c48..afd80aca1 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "time" "github.com/spf13/cobra" "golang.org/x/sync/errgroup" @@ -46,11 +47,42 @@ Exit status is 0 if the command was successful, and non-zero if there was any er }, } +type snapshotMetadata struct { + Hostname string + Time *time.Time +} + +type snapshotMetadataArgs struct { + Hostname string + Time string +} + +func (sma snapshotMetadataArgs) empty() bool { + return sma.Hostname == "" && sma.Time == "" +} + +func (sma snapshotMetadataArgs) convert() (*snapshotMetadata, error) { + if sma.empty() { + return nil, nil + } + + var timeStamp *time.Time + if sma.Time != "" { + t, err := time.ParseInLocation(TimeFormat, sma.Time, time.Local) + if err != nil { + return nil, errors.Fatalf("error in time option: %v\n", err) + } + timeStamp = &t + } + return &snapshotMetadata{Hostname: sma.Hostname, Time: timeStamp}, nil +} + // RewriteOptions collects all options for the rewrite command. type RewriteOptions struct { Forget bool DryRun bool + Metadata snapshotMetadataArgs restic.SnapshotFilter excludePatternOptions } @@ -63,11 +95,15 @@ func init() { f := cmdRewrite.Flags() f.BoolVarP(&rewriteOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones") f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done") + f.StringVar(&rewriteOptions.Metadata.Hostname, "new-host", "", "replace hostname") + f.StringVar(&rewriteOptions.Metadata.Time, "new-time", "", "replace time of the backup") initMultiSnapshotFilter(f, &rewriteOptions.SnapshotFilter, true) initExcludePatternOptions(f, &rewriteOptions.excludePatternOptions) } +type rewriteFilterFunc func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) + func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, opts RewriteOptions) (bool, error) { if sn.Tree == nil { return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str()) @@ -78,33 +114,50 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti return false, err } - selectByName := func(nodepath string) bool { - for _, reject := range rejectByNameFuncs { - if reject(nodepath) { - return false - } - } - return true + metadata, err := opts.Metadata.convert() + + if err != nil { + return false, err } - rewriter := walker.NewTreeRewriter(walker.RewriteOpts{ - RewriteNode: func(node *restic.Node, path string) *restic.Node { - if selectByName(path) { - return node + var filter rewriteFilterFunc + + if len(rejectByNameFuncs) > 0 { + selectByName := func(nodepath string) bool { + for _, reject := range rejectByNameFuncs { + if reject(nodepath) { + return false + } } - Verbosef(fmt.Sprintf("excluding %s\n", path)) - return nil - }, - DisableNodeCache: true, - }) + return true + } + + rewriter := walker.NewTreeRewriter(walker.RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { + if selectByName(path) { + return node + } + Verbosef(fmt.Sprintf("excluding %s\n", path)) + return nil + }, + DisableNodeCache: true, + }) + + filter = func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { + return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) + } + } else { + filter = func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { + return *sn.Tree, nil + } + } return filterAndReplaceSnapshot(ctx, repo, sn, - func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { - return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) - }, opts.DryRun, opts.Forget, "rewrite") + filter, opts.DryRun, opts.Forget, metadata, "rewrite") } -func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, filter func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error), dryRun bool, forget bool, addTag string) (bool, error) { +func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, + filter rewriteFilterFunc, dryRun bool, forget bool, newMetadata *snapshotMetadata, addTag string) (bool, error) { wg, wgCtx := errgroup.WithContext(ctx) repo.StartPackUploader(wgCtx, wg) @@ -128,7 +181,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r if dryRun { Verbosef("would delete empty snapshot\n") } else { - h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} + h := backend.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} if err = repo.Backend().Remove(ctx, h); err != nil { return false, err } @@ -138,7 +191,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r return true, nil } - if filteredTree == *sn.Tree { + if filteredTree == *sn.Tree && newMetadata == nil { debug.Log("Snapshot %v not modified", sn) return false, nil } @@ -151,6 +204,14 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r Verbosef("would remove old snapshot\n") } + if newMetadata != nil && newMetadata.Time != nil { + Verbosef("would set time to %s\n", newMetadata.Time) + } + + if newMetadata != nil && newMetadata.Hostname != "" { + Verbosef("would set time to %s\n", newMetadata.Hostname) + } + return true, nil } @@ -162,6 +223,16 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r sn.AddTags([]string{addTag}) } + if newMetadata != nil && newMetadata.Time != nil { + Verbosef("setting time to %s\n", *newMetadata.Time) + sn.Time = *newMetadata.Time + } + + if newMetadata != nil && newMetadata.Hostname != "" { + Verbosef("setting host to %s\n", newMetadata.Hostname) + sn.Hostname = newMetadata.Hostname + } + // Save the new snapshot. id, err := restic.SaveSnapshot(ctx, repo, sn) if err != nil { @@ -170,7 +241,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r Verbosef("saved new snapshot %v\n", id.Str()) if forget { - h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} + h := backend.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} if err = repo.Backend().Remove(ctx, h); err != nil { return false, err } @@ -181,8 +252,8 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r } func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error { - if opts.excludePatternOptions.Empty() { - return errors.Fatal("Nothing to do: no excludes provided") + if opts.excludePatternOptions.Empty() && opts.Metadata.empty() { + return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided") } repo, err := OpenRepository(ctx, gopts) @@ -207,12 +278,13 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a repo.SetDryRun() } - snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) + snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile) if err != nil { return err } - if err = repo.LoadIndex(ctx); err != nil { + bar := newIndexProgress(gopts.Quiet, gopts.JSON) + if err = repo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/cmd_rewrite_integration_test.go b/cmd/restic/cmd_rewrite_integration_test.go index e6007973b..532855f57 100644 --- a/cmd/restic/cmd_rewrite_integration_test.go +++ b/cmd/restic/cmd_rewrite_integration_test.go @@ -9,12 +9,13 @@ import ( rtest "github.com/restic/restic/internal/test" ) -func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool) { +func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool, metadata snapshotMetadataArgs) { opts := RewriteOptions{ excludePatternOptions: excludePatternOptions{ Excludes: excludes, }, - Forget: forget, + Forget: forget, + Metadata: metadata, } rtest.OK(t, runRewrite(context.TODO(), opts, gopts, nil)) @@ -38,7 +39,7 @@ func TestRewrite(t *testing.T) { createBasicRewriteRepo(t, env) // exclude some data - testRunRewriteExclude(t, env.gopts, []string{"3"}, false) + testRunRewriteExclude(t, env.gopts, []string{"3"}, false, snapshotMetadataArgs{Hostname: "", Time: ""}) snapshotIDs := testRunList(t, "snapshots", env.gopts) rtest.Assert(t, len(snapshotIDs) == 2, "expected two snapshots, got %v", snapshotIDs) testRunCheck(t, env.gopts) @@ -50,7 +51,7 @@ func TestRewriteUnchanged(t *testing.T) { snapshotID := createBasicRewriteRepo(t, env) // use an exclude that will not exclude anything - testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false) + testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false, snapshotMetadataArgs{Hostname: "", Time: ""}) newSnapshotIDs := testRunList(t, "snapshots", env.gopts) rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs) rtest.Assert(t, snapshotID == newSnapshotIDs[0], "snapshot id changed unexpectedly") @@ -63,11 +64,44 @@ func TestRewriteReplace(t *testing.T) { snapshotID := createBasicRewriteRepo(t, env) // exclude some data - testRunRewriteExclude(t, env.gopts, []string{"3"}, true) - newSnapshotIDs := testRunList(t, "snapshots", env.gopts) - rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs) + testRunRewriteExclude(t, env.gopts, []string{"3"}, true, snapshotMetadataArgs{Hostname: "", Time: ""}) + newSnapshotIDs := testListSnapshots(t, env.gopts, 1) rtest.Assert(t, snapshotID != newSnapshotIDs[0], "snapshot id should have changed") // check forbids unused blobs, thus remove them first testRunPrune(t, env.gopts, PruneOptions{MaxUnused: "0"}) testRunCheck(t, env.gopts) } + +func testRewriteMetadata(t *testing.T, metadata snapshotMetadataArgs) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + createBasicRewriteRepo(t, env) + testRunRewriteExclude(t, env.gopts, []string{}, true, metadata) + + repo, _ := OpenRepository(context.TODO(), env.gopts) + snapshots, err := restic.TestLoadAllSnapshots(context.TODO(), repo, nil) + rtest.OK(t, err) + rtest.Assert(t, len(snapshots) == 1, "expected one snapshot, got %v", len(snapshots)) + newSnapshot := snapshots[0] + + if metadata.Time != "" { + rtest.Assert(t, newSnapshot.Time.Format(TimeFormat) == metadata.Time, "New snapshot should have time %s", metadata.Time) + } + + if metadata.Hostname != "" { + rtest.Assert(t, newSnapshot.Hostname == metadata.Hostname, "New snapshot should have host %s", metadata.Hostname) + } +} + +func TestRewriteMetadata(t *testing.T) { + newHost := "new host" + newTime := "1999-01-01 11:11:11" + + for _, metadata := range []snapshotMetadataArgs{ + {Hostname: "", Time: newTime}, + {Hostname: newHost, Time: ""}, + {Hostname: newHost, Time: newTime}, + } { + testRewriteMetadata(t, metadata) + } +} diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index 889ac5e20..e94f2ed9b 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -73,7 +73,7 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions } var snapshots restic.Snapshots - for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) { + for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) { snapshots = append(snapshots, sn) } snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy) @@ -290,7 +290,7 @@ func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string) error { return nil } -// Snapshot helps to print Snaphots as JSON with their ID included. +// Snapshot helps to print Snapshots as JSON with their ID included. type Snapshot struct { *restic.Snapshot diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index a7ecd438f..a3e0cefc7 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -8,10 +8,10 @@ import ( "strings" "github.com/restic/chunker" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/restorer" "github.com/restic/restic/internal/ui" "github.com/restic/restic/internal/ui/table" "github.com/restic/restic/internal/walker" @@ -94,12 +94,12 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args } } - snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) + snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile) if err != nil { return err } - - if err = repo.LoadIndex(ctx); err != nil { + bar := newIndexProgress(gopts.Quiet, gopts.JSON) + if err = repo.LoadIndex(ctx, bar); err != nil { return err } @@ -202,8 +202,8 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest return restic.FindUsedBlobs(ctx, repo, restic.IDs{*snapshot.Tree}, stats.blobs, nil) } - uniqueInodes := make(map[uint64]struct{}) - err := walker.Walk(ctx, repo, *snapshot.Tree, restic.NewIDSet(), statsWalkTree(repo, opts, stats, uniqueInodes)) + hardLinkIndex := restorer.NewHardlinkIndex[struct{}]() + err := walker.Walk(ctx, repo, *snapshot.Tree, restic.NewIDSet(), statsWalkTree(repo, opts, stats, hardLinkIndex)) if err != nil { return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err) } @@ -211,7 +211,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest return nil } -func statsWalkTree(repo restic.Repository, opts StatsOptions, stats *statsContainer, uniqueInodes map[uint64]struct{}) walker.WalkFunc { +func statsWalkTree(repo restic.Repository, opts StatsOptions, stats *statsContainer, hardLinkIndex *restorer.HardlinkIndex[struct{}]) walker.WalkFunc { return func(parentTreeID restic.ID, npath string, node *restic.Node, nodeErr error) (bool, error) { if nodeErr != nil { return true, nodeErr @@ -270,8 +270,8 @@ func statsWalkTree(repo restic.Repository, opts StatsOptions, stats *statsContai // if inodes are present, only count each inode once // (hard links do not increase restore size) - if _, ok := uniqueInodes[node.Inode]; !ok || node.Inode == 0 { - uniqueInodes[node.Inode] = struct{}{} + if !hardLinkIndex.Has(node.Inode, node.DeviceID) || node.Inode == 0 { + hardLinkIndex.Add(node.Inode, node.DeviceID, struct{}{}) stats.TotalSize += node.Size } diff --git a/cmd/restic/cmd_tag.go b/cmd/restic/cmd_tag.go index fe4638547..01f3ad8af 100644 --- a/cmd/restic/cmd_tag.go +++ b/cmd/restic/cmd_tag.go @@ -5,6 +5,7 @@ import ( "github.com/spf13/cobra" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" @@ -12,7 +13,7 @@ import ( ) var cmdTag = &cobra.Command{ - Use: "tag [flags] [snapshot-ID ...]", + Use: "tag [flags] [snapshotID ...]", Short: "Modify tags on snapshots", Long: ` The "tag" command allows you to modify tags on exiting snapshots. @@ -20,7 +21,7 @@ The "tag" command allows you to modify tags on exiting snapshots. You can either set/replace the entire set of tags on a snapshot, or add tags to/remove tags from the existing set. -When no snapshot-ID is given, all snapshots matching the host, tag and path filter criteria are modified. +When no snapshotID is given, all snapshots matching the host, tag and path filter criteria are modified. EXIT STATUS =========== @@ -85,7 +86,7 @@ func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Sna debug.Log("new snapshot saved as %v", id) // Remove the old snapshot. - h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} + h := backend.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} if err = repo.Backend().Remove(ctx, h); err != nil { return false, err } @@ -119,7 +120,7 @@ func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []st } changeCnt := 0 - for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) { + for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) { changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten()) if err != nil { Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err) diff --git a/cmd/restic/cmd_version.go b/cmd/restic/cmd_version.go index b17103706..73469750f 100644 --- a/cmd/restic/cmd_version.go +++ b/cmd/restic/cmd_version.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "fmt" "runtime" @@ -21,8 +22,31 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("restic %s compiled with %v on %v/%v\n", - version, runtime.Version(), runtime.GOOS, runtime.GOARCH) + if globalOptions.JSON { + type jsonVersion struct { + Version string `json:"version"` + GoVersion string `json:"go_version"` + GoOS string `json:"go_os"` + GoArch string `json:"go_arch"` + } + + jsonS := jsonVersion{ + Version: version, + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + } + + err := json.NewEncoder(globalOptions.stdout).Encode(jsonS) + if err != nil { + Warnf("JSON encode failed: %v\n", err) + return + } + } else { + fmt.Printf("restic %s compiled with %v on %v/%v\n", + version, runtime.Version(), runtime.GOOS, runtime.GOARCH) + } + }, } diff --git a/cmd/restic/delete.go b/cmd/restic/delete.go index 2046ccfde..1b7937bd3 100644 --- a/cmd/restic/delete.go +++ b/cmd/restic/delete.go @@ -5,6 +5,7 @@ import ( "golang.org/x/sync/errgroup" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/restic" ) @@ -45,7 +46,7 @@ func deleteFiles(ctx context.Context, gopts GlobalOptions, ignoreError bool, rep for i := 0; i < int(workerCount); i++ { wg.Go(func() error { for id := range fileChan { - h := restic.Handle{Type: fileType, Name: id.String()} + h := backend.Handle{Type: fileType, Name: id.String()} err := repo.Backend().Remove(ctx, h) if err != nil { if !gopts.JSON { diff --git a/cmd/restic/find.go b/cmd/restic/find.go index 54d3563b1..a990b458d 100644 --- a/cmd/restic/find.go +++ b/cmd/restic/find.go @@ -3,7 +3,6 @@ package main import ( "context" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/restic" "github.com/spf13/pflag" ) @@ -33,7 +32,7 @@ func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic. out := make(chan *restic.Snapshot) go func() { defer close(out) - be, err := backend.MemorizeList(ctx, be, restic.SnapshotFile) + be, err := restic.MemorizeList(ctx, be, restic.SnapshotFile) if err != nil { Warnf("could not load snapshots: %v\n", err) return diff --git a/cmd/restic/format.go b/cmd/restic/format.go index 2f14a4575..063cd4e71 100644 --- a/cmd/restic/format.go +++ b/cmd/restic/format.go @@ -5,9 +5,10 @@ import ( "os" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui" ) -func formatNode(path string, n *restic.Node, long bool) string { +func formatNode(path string, n *restic.Node, long bool, human bool) string { if !long { return path } @@ -15,6 +16,13 @@ func formatNode(path string, n *restic.Node, long bool) string { var mode os.FileMode var target string + var size string + if human { + size = ui.FormatBytes(n.Size) + } else { + size = fmt.Sprintf("%6d", n.Size) + } + switch n.Type { case "file": mode = 0 @@ -33,8 +41,8 @@ func formatNode(path string, n *restic.Node, long bool) string { mode = os.ModeSocket } - return fmt.Sprintf("%s %5d %5d %6d %s %s%s", - mode|n.Mode, n.UID, n.GID, n.Size, + return fmt.Sprintf("%s %5d %5d %s %s %s%s", + mode|n.Mode, n.UID, n.GID, size, n.ModTime.Local().Format(TimeFormat), path, target) } diff --git a/cmd/restic/format_test.go b/cmd/restic/format_test.go new file mode 100644 index 000000000..689bd27a5 --- /dev/null +++ b/cmd/restic/format_test.go @@ -0,0 +1,61 @@ +package main + +import ( + "testing" + "time" + + "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" +) + +func TestFormatNode(t *testing.T) { + // overwrite time zone to ensure the data is formatted reproducibly + tz := time.Local + time.Local = time.UTC + defer func() { + time.Local = tz + }() + + testPath := "/test/path" + node := restic.Node{ + Name: "baz", + Type: "file", + Size: 14680064, + UID: 1000, + GID: 2000, + ModTime: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC), + } + + for _, c := range []struct { + path string + restic.Node + long bool + human bool + expect string + }{ + { + path: testPath, + Node: node, + long: false, + human: false, + expect: testPath, + }, + { + path: testPath, + Node: node, + long: true, + human: false, + expect: "---------- 1000 2000 14680064 2020-01-02 03:04:05 " + testPath, + }, + { + path: testPath, + Node: node, + long: true, + human: true, + expect: "---------- 1000 2000 14.000 MiB 2020-01-02 03:04:05 " + testPath, + }, + } { + r := formatNode(c.path, &c.Node, c.long, c.human) + rtest.Equals(t, c.expect, r) + } +} diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 9cf16cdc3..d94c5fd1b 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -44,12 +44,12 @@ import ( "golang.org/x/term" ) -var version = "0.15.2-dev (compiled manually)" +var version = "0.16.2-dev (compiled manually)" // TimeFormat is the format used for all timestamps printed by restic. const TimeFormat = "2006-01-02 15:04:05" -type backendWrapper func(r restic.Backend) (restic.Backend, error) +type backendWrapper func(r backend.Backend) (backend.Backend, error) // GlobalOptions hold all global options for restic. type GlobalOptions struct { @@ -129,7 +129,7 @@ func init() { f.StringVarP(&globalOptions.KeyHint, "key-hint", "", "", "`key` ID of key to try decrypting first (default: $RESTIC_KEY_HINT)") f.StringVarP(&globalOptions.PasswordCommand, "password-command", "", "", "shell `command` to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)") f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not output comprehensive progress report") - // use empty paremeter name as `-v, --verbose n` instead of the correct `--verbose=n` is confusing + // use empty parameter name as `-v, --verbose n` instead of the correct `--verbose=n` is confusing f.CountVarP(&globalOptions.Verbose, "verbose", "v", "be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2)") f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repository, this allows some operations on read-only repositories") f.DurationVar(&globalOptions.RetryLock, "retry-lock", 0, "retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries)") @@ -555,7 +555,7 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi func parseConfig(loc location.Location, opts options.Options) (interface{}, error) { cfg := loc.Config - if cfg, ok := cfg.(restic.ApplyEnvironmenter); ok { + if cfg, ok := cfg.(backend.ApplyEnvironmenter); ok { cfg.ApplyEnvironment("") } @@ -570,14 +570,14 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro } // Open the backend specified by a location config. -func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) { +func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (backend.Backend, error) { debug.Log("parsing location %v", location.StripPassword(gopts.backends, s)) loc, err := location.Parse(gopts.backends, s) if err != nil { return nil, errors.Fatalf("parsing repository location failed: %v", err) } - var be restic.Backend + var be backend.Backend cfg, err := parseConfig(loc, opts) if err != nil { @@ -615,7 +615,7 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio } // check if config is there - fi, err := be.Stat(ctx, restic.Handle{Type: restic.ConfigFile}) + fi, err := be.Stat(ctx, backend.Handle{Type: restic.ConfigFile}) if err != nil { return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(gopts.backends, s)) } @@ -628,7 +628,7 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio } // Create the backend specified by URI. -func create(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) { +func create(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (backend.Backend, error) { debug.Log("parsing location %v", location.StripPassword(gopts.backends, s)) loc, err := location.Parse(gopts.backends, s) if err != nil { diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go index b7cb5b333..184609d40 100644 --- a/cmd/restic/integration_helpers_test.go +++ b/cmd/restic/integration_helpers_test.go @@ -12,6 +12,7 @@ import ( "sync" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/retry" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" @@ -123,9 +124,8 @@ func directoriesContentsDiff(dir1, dir2 string) string { fmt.Fprintf(&out, "+%v\n", b.path) b = nil continue - } else { - fmt.Fprintf(&out, "%%%v\n", a.path) } + fmt.Fprintf(&out, "%%%v\n", a.path) } a, b = nil, nil @@ -205,7 +205,7 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) { extended: make(options.Options), // replace this hook with "nil" if listing a filetype more than once is necessary - backendTestHook: func(r restic.Backend) (restic.Backend, error) { return newOrderedListOnceBackend(r), nil }, + backendTestHook: func(r backend.Backend) (backend.Backend, error) { return newOrderedListOnceBackend(r), nil }, // start with default set of backends backends: globalOptions.backends, } @@ -249,7 +249,7 @@ func removePacks(gopts GlobalOptions, t testing.TB, remove restic.IDSet) { rtest.OK(t, err) for id := range remove { - rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()})) + rtest.OK(t, r.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()})) } } @@ -258,7 +258,7 @@ func removePacksExcept(gopts GlobalOptions, t testing.TB, keep restic.IDSet, rem rtest.OK(t, err) // Get all tree packs - rtest.OK(t, r.LoadIndex(context.TODO())) + rtest.OK(t, r.LoadIndex(context.TODO(), nil)) treePacks := restic.NewIDSet() r.Index().Each(context.TODO(), func(pb restic.PackedBlob) { @@ -272,7 +272,7 @@ func removePacksExcept(gopts GlobalOptions, t testing.TB, keep restic.IDSet, rem if treePacks.Has(id) != removeTreePacks || keep.Has(id) { return nil } - return r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}) + return r.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()}) })) } diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 8ea4d17d9..7cf8396a3 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -41,12 +42,12 @@ func TestCheckRestoreNoLock(t *testing.T) { // backends (like e.g. Amazon S3) as the second listing may be inconsistent to what // is expected by the first listing + some operations. type listOnceBackend struct { - restic.Backend + backend.Backend listedFileType map[restic.FileType]bool strictOrder bool } -func newListOnceBackend(be restic.Backend) *listOnceBackend { +func newListOnceBackend(be backend.Backend) *listOnceBackend { return &listOnceBackend{ Backend: be, listedFileType: make(map[restic.FileType]bool), @@ -54,7 +55,7 @@ func newListOnceBackend(be restic.Backend) *listOnceBackend { } } -func newOrderedListOnceBackend(be restic.Backend) *listOnceBackend { +func newOrderedListOnceBackend(be backend.Backend) *listOnceBackend { return &listOnceBackend{ Backend: be, listedFileType: make(map[restic.FileType]bool), @@ -62,7 +63,7 @@ func newOrderedListOnceBackend(be restic.Backend) *listOnceBackend { } } -func (be *listOnceBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *listOnceBackend) List(ctx context.Context, t restic.FileType, fn func(backend.FileInfo) error) error { if t != restic.LockFile && be.listedFileType[t] { return errors.Errorf("tried listing type %v the second time", t) } @@ -77,7 +78,7 @@ func TestListOnce(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() - env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil } pruneOpts := PruneOptions{MaxUnused: "0"} @@ -104,10 +105,10 @@ func (r *writeToOnly) WriteTo(w io.Writer) (int64, error) { } type onlyLoadWithWriteToBackend struct { - restic.Backend + backend.Backend } -func (be *onlyLoadWithWriteToBackend) Load(ctx context.Context, h restic.Handle, +func (be *onlyLoadWithWriteToBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { return be.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error { @@ -120,7 +121,7 @@ func TestBackendLoadWriteTo(t *testing.T) { defer cleanup() // setup backend which only works if it's WriteTo method is correctly propagated upwards - env.gopts.backendInnerTestHook = func(r restic.Backend) (restic.Backend, error) { + env.gopts.backendInnerTestHook = func(r backend.Backend) (backend.Backend, error) { return &onlyLoadWithWriteToBackend{Backend: r}, nil } @@ -140,7 +141,7 @@ func TestFindListOnce(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() - env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil } @@ -158,7 +159,7 @@ func TestFindListOnce(t *testing.T) { snapshotIDs := restic.NewIDSet() // specify the two oldest snapshots explicitly and use "latest" to reference the newest one - for sn := range FindFilteredSnapshots(context.TODO(), repo.Backend(), repo, &restic.SnapshotFilter{}, []string{ + for sn := range FindFilteredSnapshots(context.TODO(), repo, repo, &restic.SnapshotFilter{}, []string{ secondSnapshot[0].String(), secondSnapshot[1].String()[:8], "latest", diff --git a/cmd/restic/lock.go b/cmd/restic/lock.go index 336b56ad1..600b7476f 100644 --- a/cmd/restic/lock.go +++ b/cmd/restic/lock.go @@ -6,12 +6,14 @@ import ( "sync" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" ) type lockContext struct { + lock *restic.Lock cancel context.CancelFunc refreshWG sync.WaitGroup } @@ -104,15 +106,17 @@ retryLoop: ctx, cancel := context.WithCancel(ctx) lockInfo := &lockContext{ + lock: lock, cancel: cancel, } lockInfo.refreshWG.Add(2) refreshChan := make(chan struct{}) + forceRefreshChan := make(chan refreshLockRequest) globalLocks.Lock() globalLocks.locks[lock] = lockInfo - go refreshLocks(ctx, lock, lockInfo, refreshChan) - go monitorLockRefresh(ctx, lockInfo, refreshChan) + go refreshLocks(ctx, repo.Backend(), lockInfo, refreshChan, forceRefreshChan) + go monitorLockRefresh(ctx, lockInfo, refreshChan, forceRefreshChan) globalLocks.Unlock() return lock, ctx, err @@ -124,8 +128,13 @@ var refreshInterval = 5 * time.Minute // the difference allows to compensate for a small time drift between clients. var refreshabilityTimeout = restic.StaleLockTimeout - refreshInterval*3/2 -func refreshLocks(ctx context.Context, lock *restic.Lock, lockInfo *lockContext, refreshed chan<- struct{}) { +type refreshLockRequest struct { + result chan bool +} + +func refreshLocks(ctx context.Context, backend backend.Backend, lockInfo *lockContext, refreshed chan<- struct{}, forceRefresh <-chan refreshLockRequest) { debug.Log("start") + lock := lockInfo.lock ticker := time.NewTicker(refreshInterval) lastRefresh := lock.Time @@ -149,6 +158,22 @@ func refreshLocks(ctx context.Context, lock *restic.Lock, lockInfo *lockContext, case <-ctx.Done(): debug.Log("terminate") return + + case req := <-forceRefresh: + debug.Log("trying to refresh stale lock") + // keep on going if our current lock still exists + success := tryRefreshStaleLock(ctx, backend, lock, lockInfo.cancel) + // inform refresh goroutine about forced refresh + select { + case <-ctx.Done(): + case req.result <- success: + } + + if success { + // update lock refresh time + lastRefresh = lock.Time + } + case <-ticker.C: if time.Since(lastRefresh) > refreshabilityTimeout { // the lock is too old, wait until the expiry monitor cancels the context @@ -161,7 +186,7 @@ func refreshLocks(ctx context.Context, lock *restic.Lock, lockInfo *lockContext, Warnf("unable to refresh lock: %v\n", err) } else { lastRefresh = lock.Time - // inform monitor gorountine about successful refresh + // inform monitor goroutine about successful refresh select { case <-ctx.Done(): case refreshed <- struct{}{}: @@ -171,7 +196,7 @@ func refreshLocks(ctx context.Context, lock *restic.Lock, lockInfo *lockContext, } } -func monitorLockRefresh(ctx context.Context, lockInfo *lockContext, refreshed <-chan struct{}) { +func monitorLockRefresh(ctx context.Context, lockInfo *lockContext, refreshed <-chan struct{}, forceRefresh chan<- refreshLockRequest) { // time.Now() might use a monotonic timer which is paused during standby // convert to unix time to ensure we compare real time values lastRefresh := time.Now().UnixNano() @@ -183,24 +208,47 @@ func monitorLockRefresh(ctx context.Context, lockInfo *lockContext, refreshed <- // timers are paused during standby, which is a problem as the refresh timeout // _must_ expire if the host was too long in standby. Thus fall back to periodic checks // https://github.com/golang/go/issues/35012 - timer := time.NewTimer(pollDuration) + ticker := time.NewTicker(pollDuration) defer func() { - timer.Stop() + ticker.Stop() lockInfo.cancel() lockInfo.refreshWG.Done() }() + var refreshStaleLockResult chan bool + for { select { case <-ctx.Done(): debug.Log("terminate expiry monitoring") return case <-refreshed: + if refreshStaleLockResult != nil { + // ignore delayed refresh notifications while the stale lock is refreshed + continue + } lastRefresh = time.Now().UnixNano() - case <-timer.C: - if time.Now().UnixNano()-lastRefresh < refreshabilityTimeout.Nanoseconds() { - // restart timer - timer.Reset(pollDuration) + case <-ticker.C: + if time.Now().UnixNano()-lastRefresh < refreshabilityTimeout.Nanoseconds() || refreshStaleLockResult != nil { + continue + } + + debug.Log("trying to refreshStaleLock") + // keep on going if our current lock still exists + refreshReq := refreshLockRequest{ + result: make(chan bool), + } + refreshStaleLockResult = refreshReq.result + + // inform refresh goroutine about forced refresh + select { + case <-ctx.Done(): + case forceRefresh <- refreshReq: + } + case success := <-refreshStaleLockResult: + if success { + lastRefresh = time.Now().UnixNano() + refreshStaleLockResult = nil continue } @@ -210,6 +258,25 @@ func monitorLockRefresh(ctx context.Context, lockInfo *lockContext, refreshed <- } } +func tryRefreshStaleLock(ctx context.Context, be backend.Backend, lock *restic.Lock, cancel context.CancelFunc) bool { + freeze := backend.AsBackend[backend.FreezeBackend](be) + if freeze != nil { + debug.Log("freezing backend") + freeze.Freeze() + defer freeze.Unfreeze() + } + + err := lock.RefreshStaleLock(ctx) + if err != nil { + Warnf("failed to refresh stale lock: %v\n", err) + // cancel context while the backend is still frozen to prevent accidental modifications + cancel() + return false + } + + return true +} + func unlockRepo(lock *restic.Lock) { if lock == nil { return diff --git a/cmd/restic/lock_test.go b/cmd/restic/lock_test.go index 150bd8730..bf22db699 100644 --- a/cmd/restic/lock_test.go +++ b/cmd/restic/lock_test.go @@ -5,16 +5,27 @@ import ( "fmt" "runtime" "strings" + "sync" "testing" "time" + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/mem" + "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) -func openTestRepo(t *testing.T, wrapper backendWrapper) (*repository.Repository, func(), *testEnvironment) { +func openLockTestRepo(t *testing.T, wrapper backendWrapper) (*repository.Repository, func(), *testEnvironment) { env, cleanup := withTestEnvironment(t) + + reg := location.NewRegistry() + reg.Register(mem.NewFactory()) + env.gopts.backends = reg + env.gopts.Repo = "mem:" + if wrapper != nil { env.gopts.backendTestHook = wrapper } @@ -36,7 +47,7 @@ func checkedLockRepo(ctx context.Context, t *testing.T, repo restic.Repository, } func TestLock(t *testing.T) { - repo, cleanup, env := openTestRepo(t, nil) + repo, cleanup, env := openLockTestRepo(t, nil) defer cleanup() lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, env) @@ -47,7 +58,7 @@ func TestLock(t *testing.T) { } func TestLockCancel(t *testing.T) { - repo, cleanup, env := openTestRepo(t, nil) + repo, cleanup, env := openLockTestRepo(t, nil) defer cleanup() ctx, cancel := context.WithCancel(context.Background()) @@ -63,7 +74,7 @@ func TestLockCancel(t *testing.T) { } func TestLockUnlockAll(t *testing.T) { - repo, cleanup, env := openTestRepo(t, nil) + repo, cleanup, env := openLockTestRepo(t, nil) defer cleanup() lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, env) @@ -78,7 +89,7 @@ func TestLockUnlockAll(t *testing.T) { } func TestLockConflict(t *testing.T) { - repo, cleanup, env := openTestRepo(t, nil) + repo, cleanup, env := openLockTestRepo(t, nil) defer cleanup() repo2, err := OpenRepository(context.TODO(), env.gopts) test.OK(t, err) @@ -94,11 +105,11 @@ func TestLockConflict(t *testing.T) { } type writeOnceBackend struct { - restic.Backend + backend.Backend written bool } -func (b *writeOnceBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (b *writeOnceBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { if b.written { return fmt.Errorf("fail after first write") } @@ -107,7 +118,7 @@ func (b *writeOnceBackend) Save(ctx context.Context, h restic.Handle, rd restic. } func TestLockFailedRefresh(t *testing.T) { - repo, cleanup, env := openTestRepo(t, func(r restic.Backend) (restic.Backend, error) { + repo, cleanup, env := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) { return &writeOnceBackend{Backend: r}, nil }) defer cleanup() @@ -133,11 +144,11 @@ func TestLockFailedRefresh(t *testing.T) { } type loggingBackend struct { - restic.Backend + backend.Backend t *testing.T } -func (b *loggingBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (b *loggingBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { b.t.Logf("save %v @ %v", h, time.Now()) err := b.Backend.Save(ctx, h, rd) b.t.Logf("save finished %v @ %v", h, time.Now()) @@ -145,7 +156,7 @@ func (b *loggingBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re } func TestLockSuccessfulRefresh(t *testing.T) { - repo, cleanup, env := openTestRepo(t, func(r restic.Backend) (restic.Backend, error) { + repo, cleanup, env := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) { return &loggingBackend{ Backend: r, t: t, @@ -182,8 +193,71 @@ func TestLockSuccessfulRefresh(t *testing.T) { unlockRepo(lock) } +type slowBackend struct { + backend.Backend + m sync.Mutex + sleep time.Duration +} + +func (b *slowBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { + b.m.Lock() + sleep := b.sleep + b.m.Unlock() + time.Sleep(sleep) + return b.Backend.Save(ctx, h, rd) +} + +func TestLockSuccessfulStaleRefresh(t *testing.T) { + var sb *slowBackend + repo, cleanup, env := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) { + sb = &slowBackend{Backend: r} + return sb, nil + }) + defer cleanup() + + t.Logf("test for successful lock refresh %v", time.Now()) + // reduce locking intervals to be suitable for testing + ri, rt := refreshInterval, refreshabilityTimeout + refreshInterval = 10 * time.Millisecond + refreshabilityTimeout = 50 * time.Millisecond + defer func() { + refreshInterval, refreshabilityTimeout = ri, rt + }() + + lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, env) + // delay lock refreshing long enough that the lock would expire + sb.m.Lock() + sb.sleep = refreshabilityTimeout + refreshInterval + sb.m.Unlock() + + select { + case <-wrappedCtx.Done(): + // don't call t.Fatal to allow the lock to be properly cleaned up + t.Error("lock refresh failed", time.Now()) + + case <-time.After(refreshabilityTimeout): + } + // reset slow backend + sb.m.Lock() + sb.sleep = 0 + sb.m.Unlock() + debug.Log("normal lock period has expired") + + select { + case <-wrappedCtx.Done(): + // don't call t.Fatal to allow the lock to be properly cleaned up + t.Error("lock refresh failed", time.Now()) + + case <-time.After(3 * refreshabilityTimeout): + // expected lock refresh to work + } + + // unlockRepo should not crash + unlockRepo(lock) +} + func TestLockWaitTimeout(t *testing.T) { - repo, cleanup, env := openTestRepo(t, nil) + repo, cleanup, env := openLockTestRepo(t, nil) defer cleanup() elock, _, err := lockRepoExclusive(context.TODO(), repo, env.gopts.RetryLock, env.gopts.JSON) @@ -205,8 +279,9 @@ func TestLockWaitTimeout(t *testing.T) { test.OK(t, lock.Unlock()) test.OK(t, elock.Unlock()) } + func TestLockWaitCancel(t *testing.T) { - repo, cleanup, env := openTestRepo(t, nil) + repo, cleanup, env := openLockTestRepo(t, nil) defer cleanup() elock, _, err := lockRepoExclusive(context.TODO(), repo, env.gopts.RetryLock, env.gopts.JSON) @@ -215,10 +290,10 @@ func TestLockWaitCancel(t *testing.T) { retryLock := 200 * time.Millisecond cancelAfter := 40 * time.Millisecond + start := time.Now() ctx, cancel := context.WithCancel(context.TODO()) time.AfterFunc(cancelAfter, cancel) - start := time.Now() lock, _, err := lockRepo(ctx, repo, retryLock, env.gopts.JSON) duration := time.Since(start) @@ -227,14 +302,14 @@ func TestLockWaitCancel(t *testing.T) { test.Assert(t, strings.Contains(err.Error(), "context canceled"), "create normal lock with exclusively locked repo didn't return the correct error") test.Assert(t, cancelAfter <= duration && duration < retryLock-10*time.Millisecond, - "create normal lock with exclusively locked repo didn't return in time") + "create normal lock with exclusively locked repo didn't return in time, duration %v", duration) test.OK(t, lock.Unlock()) test.OK(t, elock.Unlock()) } func TestLockWaitSuccess(t *testing.T) { - repo, cleanup, env := openTestRepo(t, nil) + repo, cleanup, env := openLockTestRepo(t, nil) defer cleanup() elock, _, err := lockRepoExclusive(context.TODO(), repo, env.gopts.RetryLock, env.gopts.JSON) diff --git a/cmd/restic/main.go b/cmd/restic/main.go index 64b75b43a..4595e8161 100644 --- a/cmd/restic/main.go +++ b/cmd/restic/main.go @@ -9,15 +9,20 @@ import ( "runtime" godebug "runtime/debug" + "github.com/spf13/cobra" + "go.uber.org/automaxprocs/maxprocs" + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" "github.com/restic/restic/internal/restic" - - "github.com/spf13/cobra" - - "github.com/restic/restic/internal/errors" ) +func init() { + // don't import `go.uber.org/automaxprocs` to disable the log output + _, _ = maxprocs.Set() +} + // cmdRoot is the base command when no other command has been specified. var cmdRoot = &cobra.Command{ Use: "restic", @@ -25,6 +30,8 @@ var cmdRoot = &cobra.Command{ Long: ` restic is a backup program which allows saving multiple revisions of files and directories in an encrypted repository stored on different backends. + +The full documentation can be found at https://restic.readthedocs.io/ . `, SilenceErrors: true, SilenceUsage: true, @@ -73,7 +80,7 @@ directories in an encrypted repository stored on different backends. // user for authentication). func needsPassword(cmd string) bool { switch cmd { - case "cache", "generate", "help", "options", "self-update", "version": + case "cache", "generate", "help", "options", "self-update", "version", "__complete": return false default: return true diff --git a/cmd/restic/progress.go b/cmd/restic/progress.go index 4b6025a54..8b33f94c9 100644 --- a/cmd/restic/progress.go +++ b/cmd/restic/progress.go @@ -29,13 +29,12 @@ func calculateProgressInterval(show bool, json bool) time.Duration { return interval } -// newProgressMax returns a progress.Counter that prints to stdout. -func newProgressMax(show bool, max uint64, description string) *progress.Counter { +// newTerminalProgressMax returns a progress.Counter that prints to stdout or terminal if provided. +func newGenericProgressMax(show bool, max uint64, description string, print func(status string)) *progress.Counter { if !show { return nil } interval := calculateProgressInterval(show, false) - canUpdateStatus := stdoutCanUpdateStatus() return progress.NewCounter(interval, max, func(v uint64, max uint64, d time.Duration, final bool) { var status string @@ -47,14 +46,28 @@ func newProgressMax(show bool, max uint64, description string) *progress.Counter ui.FormatDuration(d), ui.FormatPercent(v, max), v, max, description) } - printProgress(status, canUpdateStatus) + print(status) if final { fmt.Print("\n") } }) } -func printProgress(status string, canUpdateStatus bool) { +func newTerminalProgressMax(show bool, max uint64, description string, term *termstatus.Terminal) *progress.Counter { + return newGenericProgressMax(show, max, description, func(status string) { + term.SetStatus([]string{status}) + }) +} + +// newProgressMax calls newTerminalProgress without a terminal (print to stdout) +func newProgressMax(show bool, max uint64, description string) *progress.Counter { + return newGenericProgressMax(show, max, description, printProgress) +} + +func printProgress(status string) { + + canUpdateStatus := stdoutCanUpdateStatus() + w := stdoutTerminalWidth() if w > 0 { if w < 3 { @@ -83,3 +96,11 @@ func printProgress(status string, canUpdateStatus bool) { _, _ = os.Stdout.Write([]byte(clear + status + carriageControl)) } + +func newIndexProgress(quiet bool, json bool) *progress.Counter { + return newProgressMax(!quiet && !json && stdoutIsTerminal(), 0, "index files loaded") +} + +func newIndexTerminalProgress(quiet bool, json bool, term *termstatus.Terminal) *progress.Counter { + return newTerminalProgressMax(!quiet && !json && stdoutIsTerminal(), 0, "index files loaded", term) +} diff --git a/doc/010_introduction.rst b/doc/010_introduction.rst index 5c213f6cd..e6bffdea1 100644 --- a/doc/010_introduction.rst +++ b/doc/010_introduction.rst @@ -17,3 +17,47 @@ Introduction Restic is a fast and secure backup program. In the following sections, we will present typical workflows, starting with installing, preparing a new repository, and making the first backup. + +Quickstart Guide +**************** + +To get started with a local repository, first define some environment variables: + +.. code-block:: console + + export RESTIC_REPOSITORY=/srv/restic-repo + export RESTIC_PASSWORD=some-strong-password + +Initialize the repository (first time only): + +.. code-block:: console + + restic init + +Create your first backup: + +.. code-block:: console + + restic backup ~/work + +You can list all the snapshots you created with: + +.. code-block:: console + + restic snapshots + +You can restore a backup by noting the snapshot ID you want and running: + +.. code-block:: console + + restic restore --target /tmp/restore-work your-snapshot-ID + +It is a good idea to periodically check your repository's metadata: + +.. code-block:: console + + restic check + # or full data: + restic check --read-data + +For more details continue reading the next sections. diff --git a/doc/020_installation.rst b/doc/020_installation.rst index a39ae91e9..0f1cd6c04 100644 --- a/doc/020_installation.rst +++ b/doc/020_installation.rst @@ -84,6 +84,12 @@ If you are using macOS, you can install restic using the $ brew install restic +On Linux and macOS, you can also install it using `pkgx `__: + +.. code-block:: console + + $ pkgx install restic + You may also install it using `MacPorts `__: .. code-block:: console @@ -279,7 +285,7 @@ From Source *********** restic is written in the Go programming language and you need at least -Go version 1.18. Building for Solaris requires at least Go version 1.20. +Go version 1.19. Building for Solaris requires at least Go version 1.20. Building restic may also work with older versions of Go, but that's not supported. See the `Getting started `__ guide of the Go project for diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index 8c166c79b..6c89e356c 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -119,10 +119,10 @@ user's home directory. Also, if the SFTP server is enforcing domain-confined users, you can specify the user this way: ``user@domain@host``. -.. note:: Please be aware that sftp servers do not expand the tilde character +.. note:: Please be aware that SFTP servers do not expand the tilde character (``~``) normally used as an alias for a user's home directory. If you want to specify a path relative to the user's home directory, pass a - relative path to the sftp backend. + relative path to the SFTP backend. If you need to specify a port number or IPv6 address, you'll need to use URL syntax. E.g., the repository ``/srv/restic-repo`` on ``[::1]`` (localhost) @@ -172,9 +172,11 @@ Then use it in the backend specification: Last, if you'd like to use an entirely different program to create the SFTP connection, you can specify the command to be run with the option -``-o sftp.command="foobar"``. +``-o sftp.command="foobar"``. Alternatively, ``-o sftp.args`` allows +setting the arguments passed to the default SSH command (ignored when +``sftp.command`` is set) -.. note:: Please be aware that sftp servers close connections when no data is +.. note:: Please be aware that SFTP servers close connections when no data is received by the client. This can happen when restic is processing huge amounts of unchanged data. To avoid this issue add the following lines to the client's .ssh/config file: @@ -209,6 +211,14 @@ are some more examples: $ restic -r rest:https://user:pass@host:8000/ init $ restic -r rest:https://user:pass@host:8000/my_backup_repo/ init +The server username and password can be specified using environment +variables as well: + +.. code-block:: console + + $ export RESTIC_REST_USERNAME= + $ export RESTIC_REST_PASSWORD= + If you use TLS, restic will use the system's CA certificates to verify the server certificate. When the verification fails, restic refuses to proceed and exits with an error. If you have your own self-signed certificate, or a custom @@ -321,7 +331,7 @@ Wasabi ************ `Wasabi `__ is a low cost Amazon S3 conformant object storage provider. -Due to it's S3 conformance, Wasabi can be used as a storage provider for a restic repository. +Due to its S3 conformance, Wasabi can be used as a storage provider for a restic repository. - Create a Wasabi bucket using the `Wasabi Console `__. - Determine the correct Wasabi service URL for your bucket `here `__. @@ -840,7 +850,7 @@ To make this work we can employ the help of the ``setgid`` permission bit available on Linux and most other Unix systems. This permission bit makes newly created directories inherit both the group owner (gid) and setgid bit from the parent directory. Setting this bit requires root but since it -propagates down to any new directories we only have to do this priviledged +propagates down to any new directories we only have to do this privileged setup once: .. code-block:: console diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 29e7cf3b0..9f66547bc 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -146,7 +146,7 @@ change detection rule based on file metadata to determine whether a file is likely unchanged since a previous backup. If it is, the file is not scanned again. -The previous backup snapshot, called "parent" snaphot in restic terminology, +The previous backup snapshot, called "parent" snapshot in restic terminology, is determined as follows. By default restic groups snapshots by hostname and backup paths, and then selects the latest snapshot in the group that matches the current backup. You can change the selection criteria using the @@ -451,6 +451,15 @@ and displays a small statistic, just pass the command two snapshot IDs: Added: 16.403 MiB Removed: 16.402 MiB +To only compare files in specific subfolders, you can use the ``:`` +syntax, where ``snapshot`` is the ID of a snapshot (or the string ``latest``) and ``subfolder`` +is a path within the snapshot. For example, to only compare files in the ``/restic`` +folder, you could use the following command: + +.. code-block:: console + + $ restic -r /srv/restic-repo diff 5845b002:/restic 2ab627a6:/restic + Backing up special items and metadata ************************************* @@ -480,35 +489,71 @@ particular note are:: - file ownership and ACLs on Windows - the "hidden" flag on Windows + +Reading data from a command +*************************** + +Sometimes, it can be useful to directly save the output of a program, for example, +``mysqldump`` so that the SQL can later be restored. Restic supports this mode +of operation; just supply the option ``--stdin-from-command`` when using the +``backup`` action, and write the command in place of the files/directories: + +.. code-block:: console + + $ restic -r /srv/restic-repo backup --stdin-from-command mysqldump [...] + +This command creates a new snapshot based on the standard output of ``mysqldump``. +By default, the command's standard output is saved in a file named ``stdin``. +A different name can be specified with ``--stdin-filename``: + +.. code-block:: console + + $ restic -r /srv/restic-repo backup --stdin-filename production.sql --stdin-from-command mysqldump [...] + +Restic uses the command exit code to determine whether the command succeeded. A +non-zero exit code from the command causes restic to cancel the backup. This causes +restic to fail with exit code 1. No snapshot will be created in this case. + + Reading data from stdin *********************** -Sometimes it can be nice to directly save the output of a program, e.g. -``mysqldump`` so that the SQL can later be restored. Restic supports -this mode of operation, just supply the option ``--stdin`` to the -``backup`` command like this: +.. warning:: + + Restic cannot detect if data read from stdin is complete or not. As explained + below, this can cause incomplete backup unless additional checks (outside of + restic) are configured. If possible, use ``--stdin-from-command`` instead. + +Alternatively, restic supports reading arbitrary data directly from the standard +input. Use the option ``--stdin`` of the ``backup`` command as follows: .. code-block:: console - $ set -o pipefail + # Will not notice failures, see the warning below + $ gzip bigfile.dat | restic -r /srv/restic-repo backup --stdin + +This creates a new snapshot of the content of ``bigfile.dat``. +As for ``--stdin-from-command``, the default file name is ``stdin``; a +different name can be specified with ``--stdin-filename``. + +**Important**: while it is possible to pipe a command output to restic using +``--stdin``, doing so is discouraged as it will mask errors from the +command, leading to corrupted backups. For example, in the following code +block, if ``mysqldump`` fails to connect to the MySQL database, the restic +backup will nevertheless succeed in creating an _empty_ backup: + +.. code-block:: console + + # Will not notice failures, read the warning above $ mysqldump [...] | restic -r /srv/restic-repo backup --stdin -This creates a new snapshot of the output of ``mysqldump``. You can then -use e.g. the fuse mounting option (see below) to mount the repository -and read the file. - -By default, the file name ``stdin`` is used, a different name can be -specified with ``--stdin-filename``, e.g. like this: - -.. code-block:: console - - $ mysqldump [...] | restic -r /srv/restic-repo backup --stdin --stdin-filename production.sql - -The option ``pipefail`` is highly recommended so that a non-zero exit code from -one of the programs in the pipe (e.g. ``mysqldump`` here) makes the whole chain -return a non-zero exit code. Refer to the `Use the Unofficial Bash Strict Mode -`__ for more -details on this. +A simple solution is to use ``--stdin-from-command`` (see above). If you +still need to use the ``--stdin`` flag, you must use the shell option ``set -o pipefail`` +(so that a non-zero exit code from one of the programs in the pipe makes the +whole chain return a non-zero exit code) and you must check the exit code of +the pipe and act accordingly (e.g., remove the last backup). Refer to the +`Use the Unofficial Bash Strict Mode `__ +for more details on this. Tags for backup @@ -584,9 +629,16 @@ environment variables. The following lists these environment variables: AWS_PROFILE Amazon credentials profile (alternative to specifying key and region) AWS_SHARED_CREDENTIALS_FILE Location of the AWS CLI shared credentials file (default: ~/.aws/credentials) - ST_AUTH Auth URL for keystone v1 authentication - ST_USER Username for keystone v1 authentication - ST_KEY Password for keystone v1 authentication + AZURE_ACCOUNT_NAME Account name for Azure + AZURE_ACCOUNT_KEY Account key for Azure + AZURE_ACCOUNT_SAS Shared access signatures (SAS) for Azure + AZURE_ENDPOINT_SUFFIX Endpoint suffix for Azure Storage (default: core.windows.net) + + B2_ACCOUNT_ID Account ID or applicationKeyId for Backblaze B2 + B2_ACCOUNT_KEY Account Key or applicationKey for Backblaze B2 + + GOOGLE_PROJECT_ID Project ID for Google Cloud Storage + GOOGLE_APPLICATION_CREDENTIALS Application Credentials for Google Cloud Storage (e.g. $HOME/.config/gs-secret-restic-key.json) OS_AUTH_URL Auth URL for keystone authentication OS_REGION_NAME Region name for keystone authentication @@ -610,23 +662,19 @@ environment variables. The following lists these environment variables: OS_STORAGE_URL Storage URL for token authentication OS_AUTH_TOKEN Auth token for token authentication - B2_ACCOUNT_ID Account ID or applicationKeyId for Backblaze B2 - B2_ACCOUNT_KEY Account Key or applicationKey for Backblaze B2 - - AZURE_ACCOUNT_NAME Account name for Azure - AZURE_ACCOUNT_KEY Account key for Azure - AZURE_ACCOUNT_SAS Shared access signatures (SAS) for Azure - AZURE_ENDPOINT_SUFFIX Endpoint suffix for Azure Storage (default: core.windows.net) - - GOOGLE_PROJECT_ID Project ID for Google Cloud Storage - GOOGLE_APPLICATION_CREDENTIALS Application Credentials for Google Cloud Storage (e.g. $HOME/.config/gs-secret-restic-key.json) - RESTIC_SMB_USER SMB user for NTLM authentication RESTIC_SMB_PASSWORD SMB password for NTLM authentication RESTIC_SMB_DOMAIN DOMAIN for SMB authentication RCLONE_BWLIMIT rclone bandwidth limit + RESTIC_REST_USERNAME Restic REST Server username + RESTIC_REST_PASSWORD Restic REST Server password + + ST_AUTH Auth URL for keystone v1 authentication + ST_USER Username for keystone v1 authentication + ST_KEY Password for keystone v1 authentication + See :ref:`caching` for the rules concerning cache locations when ``RESTIC_CACHE_DIR`` is not set. diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index 82a20bac4..68c181fa2 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -120,7 +120,7 @@ be skipped by later copy runs. The source repository is specified with ``--from-repo`` or can be read from a file specified via ``--from-repository-file``. Both of these options can also be set as environment variables ``$RESTIC_FROM_REPOSITORY`` or -``$RESTIC_FROM_REPOSITORY_FILE``, respectively. For the destination repository +``$RESTIC_FROM_REPOSITORY_FILE``, respectively. For the source repository the password can be read from a file ``--from-password-file`` or from a command ``--from-password-command``. Alternatively the environment variables ``$RESTIC_FROM_PASSWORD_COMMAND`` and @@ -136,6 +136,8 @@ or the environment variable ``$RESTIC_FROM_KEY_HINT``. repository. You can avoid this limitation by using the rclone backend along with remotes which are configured in rclone. +.. note:: If `copy` is aborted, `copy` will resume the interrupted copying when it is run again. It's possible that up to 10 minutes of progress can be lost because the repository index is only updated from time to time. + .. _copy-filtering-snapshots: Filtering snapshots to copy @@ -232,6 +234,27 @@ modifying the repository. Instead restic will only print the actions it would perform. +Modifying metadata of snapshots +=============================== + +Sometimes it may be desirable to change the metadata of an existing snapshot. +Currently, rewriting the hostname and the time of the backup is supported. +This is possible using the ``rewrite`` command with the option ``--new-host`` followed by the desired new hostname or the option ``--new-time`` followed by the desired new timestamp. + +.. code-block:: console + $ restic rewrite --new-host newhost --new-time "1999-01-01 11:11:11" + + repository b7dbade3 opened (version 2, compression level auto) + [0:00] 100.00% 1 / 1 index files loaded + + snapshot 8ed674f4 of [/path/to/abc.txt] at 2023-11-27 21:57:52.439139291 +0100 CET) + setting time to 1999-01-01 11:11:11 +0100 CET + setting host to newhost + saved new snapshot c05da643 + + modified 1 snapshots + + .. _checking-integrity: Checking integrity and consistency @@ -335,7 +358,7 @@ over 5 separate invocations: $ restic -r /srv/restic-repo check --read-data-subset=4/5 $ restic -r /srv/restic-repo check --read-data-subset=5/5 -Use ``--read-data-subset=x%`` to check a randomly choosen subset of the +Use ``--read-data-subset=x%`` to check a randomly chosen subset of the repository pack files. It takes one parameter, ``x``, the percentage of pack files to check as an integer or floating point number. This will not guarantee to cover all available pack files after sufficient runs, but it is diff --git a/doc/050_restore.rst b/doc/050_restore.rst index 47a1be003..56f6458ed 100644 --- a/doc/050_restore.rst +++ b/doc/050_restore.rst @@ -48,6 +48,18 @@ files in the snapshot. For example, to restore a single file: This will restore the file ``foo`` to ``/tmp/restore-work/work/foo``. +To only restore a specific subfolder, you can use the ``:`` +syntax, where ``snapshot`` is the ID of a snapshot (or the string ``latest``) +and ``subfolder`` is a path within the snapshot. + +.. code-block:: console + + $ restic -r /srv/restic-repo restore 79766175:/work --target /tmp/restore-work --include /foo + enter password for repository: + restoring to /tmp/restore-work + +This will restore the file ``foo`` to ``/tmp/restore-work/foo``. + You can use the command ``restic ls latest`` or ``restic find foo`` to find the path to the file within the snapshot. This path you can then pass to ``--include`` in verbatim to only restore the single file or directory. @@ -94,7 +106,7 @@ Restic supports storage and preservation of hard links. However, since hard links exist in the scope of a filesystem by definition, restoring hard links from a fuse mount should be done by a program that preserves hard links. A program that does so is ``rsync``, used with the option ---hard-links. +``--hard-links``. .. note:: ``restic mount`` is mostly useful if you want to restore just a few files out of a snapshot, or to check which files are contained in a snapshot. @@ -151,8 +163,14 @@ output the contents in the tar (default) or zip format: .. code-block:: console $ restic -r /srv/restic-repo dump latest /home/other/work > restore.tar - + .. code-block:: console $ restic -r /srv/restic-repo dump -a zip latest /home/other/work > restore.zip +The folder content is then contained at ``/home/other/work`` within the archive. +To include the folder content at the root of the archive, you can use the ``:`` syntax: + +.. code-block:: console + + $ restic -r /srv/restic-repo dump latest:/home/other/work / > restore.tar diff --git a/doc/070_encryption.rst b/doc/070_encryption.rst index dc651cc07..f18f0cd41 100644 --- a/doc/070_encryption.rst +++ b/doc/070_encryption.rst @@ -49,3 +49,5 @@ per repository. In fact, you can use the ``list``, ``add``, ``remove``, and ---------------------------------------------------------------------- 5c657874 username kasimir 2015-08-12 13:35:05 *eb78040b username kasimir 2015-08-12 13:29:57 + +Note that the currently used key is indicated by an asterisk (``*``). diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index a4b983d7c..d94074232 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -145,7 +145,7 @@ Verbose status provides details about the progress, including details about back Summary ^^^^^^^ -Summary is the last output line in a successful backup. +Summary is the last output line in a successful backup. +---------------------------+---------------------------------------------------------+ | ``message_type`` | Always "summary" | @@ -277,6 +277,8 @@ Match object +-----------------+----------------------------------------------+ | ``group`` | Name of group | +-----------------+----------------------------------------------+ +| ``inode`` | Inode number | ++-----------------+----------------------------------------------+ | ``mode`` | UNIX file mode, shorthand of ``permissions`` | +-----------------+----------------------------------------------+ | ``device_id`` | OS specific device identifier | @@ -470,6 +472,8 @@ node +-----------------+--------------------------+ | ``ctime`` | Node creation time | +-----------------+--------------------------+ +| ``inode`` | Inode number of node | ++-----------------+--------------------------+ restore @@ -485,7 +489,7 @@ Status +----------------------+------------------------------------------------------------+ |``seconds_elapsed`` | Time since restore started | +----------------------+------------------------------------------------------------+ -|``percent_done`` | Percentage of data backed up (bytes_restored/total_bytes) | +|``percent_done`` | Percentage of data restored (bytes_restored/total_bytes) | +----------------------+------------------------------------------------------------+ |``total_files`` | Total number of files detected | +----------------------+------------------------------------------------------------+ @@ -552,7 +556,7 @@ The snapshots command returns a single JSON object, an array with objects of the stats ----- -The snapshots command returns a single JSON object. +The stats command returns a single JSON object. +------------------------------+-----------------------------------------------------+ | ``total_size`` | Repository size in bytes | @@ -572,3 +576,19 @@ The snapshots command returns a single JSON object. +------------------------------+-----------------------------------------------------+ | ``compression_space_saving`` | Overall space saving due to compression | +------------------------------+-----------------------------------------------------+ + + +version +------- + +The version command returns a single JSON object. + ++----------------+--------------------+ +| ``version`` | restic version | ++----------------+--------------------+ +| ``go_version`` | Go compile version | ++----------------+--------------------+ +| ``go_os`` | Go OS | ++----------------+--------------------+ +| ``go_arch`` | Go architecture | ++----------------+--------------------+ diff --git a/doc/080_examples.rst b/doc/080_examples.rst index 74ce3c0b3..a31af5686 100644 --- a/doc/080_examples.rst +++ b/doc/080_examples.rst @@ -220,7 +220,7 @@ repository: .. code-block:: console - $ ./restic init + $ restic init created restic backend b5c661a86a at s3:https://s3.amazonaws.com/restic-demo Please note that knowledge of your password is required to access @@ -236,14 +236,14 @@ restic is now ready to be used with Amazon S3. Try to create a backup: 10+0 records out 10485760 bytes (10 MB, 10 MiB) copied, 0,0891322 s, 118 MB/s - $ ./restic backup test.bin + $ restic backup test.bin scan [/home/philip/restic-demo/test.bin] scanned 0 directories, 1 files in 0:00 [0:04] 100.00% 2.500 MiB/s 10.000 MiB / 10.000 MiB 1 / 1 items ... ETA 0:00 duration: 0:04, 2.47MiB/s snapshot 10fdbace saved - $ ./restic snapshots + $ restic snapshots ID Date Host Tags Directory ---------------------------------------------------------------------- 10fdbace 2017-03-26 16:41:50 blackbox /home/philip/restic-demo/test.bin @@ -252,7 +252,7 @@ A snapshot was created and stored in the S3 bucket. By default backups to Amazon .. code-block:: console - $ ./restic backup -o s3.storage-class=REDUCED_REDUNDANCY test.bin + $ restic backup -o s3.storage-class=REDUCED_REDUNDANCY test.bin This snapshot may now be restored: @@ -260,7 +260,7 @@ This snapshot may now be restored: $ mkdir restore - $ ./restic restore 10fdbace --target restore + $ restic restore 10fdbace --target restore restoring to restore $ ls restore/ diff --git a/doc/bash-completion.sh b/doc/bash-completion.sh index 42f459f65..e691af363 100644 --- a/doc/bash-completion.sh +++ b/doc/bash-completion.sh @@ -414,6 +414,12 @@ _restic_backup() flags+=("-f") local_nonpersistent_flags+=("--force") local_nonpersistent_flags+=("-f") + flags+=("--group-by=") + two_word_flags+=("--group-by") + two_word_flags+=("-g") + local_nonpersistent_flags+=("--group-by") + local_nonpersistent_flags+=("--group-by=") + local_nonpersistent_flags+=("-g") flags+=("--help") flags+=("-h") local_nonpersistent_flags+=("--help") @@ -500,6 +506,8 @@ _restic_backup() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -570,6 +578,8 @@ _restic_cache() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -632,6 +642,8 @@ _restic_cat() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -702,6 +714,8 @@ _restic_check() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -798,6 +812,8 @@ _restic_copy() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -862,6 +878,8 @@ _restic_diff() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -944,6 +962,8 @@ _restic_dump() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -980,6 +1000,8 @@ _restic_find() local_nonpersistent_flags+=("--host") local_nonpersistent_flags+=("--host=") local_nonpersistent_flags+=("-H") + flags+=("--human-readable") + local_nonpersistent_flags+=("--human-readable") flags+=("--ignore-case") flags+=("-i") local_nonpersistent_flags+=("--ignore-case") @@ -1054,6 +1076,8 @@ _restic_find() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1222,6 +1246,8 @@ _restic_forget() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1304,6 +1330,8 @@ _restic_generate() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1362,6 +1390,8 @@ _restic_help() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1451,6 +1481,8 @@ _restic_init() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1525,6 +1557,8 @@ _restic_key() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1587,6 +1621,8 @@ _restic_list() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1621,6 +1657,8 @@ _restic_ls() local_nonpersistent_flags+=("--host") local_nonpersistent_flags+=("--host=") local_nonpersistent_flags+=("-H") + flags+=("--human-readable") + local_nonpersistent_flags+=("--human-readable") flags+=("--long") flags+=("-l") local_nonpersistent_flags+=("--long") @@ -1669,6 +1707,8 @@ _restic_ls() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1735,6 +1775,8 @@ _restic_migrate() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1825,6 +1867,8 @@ _restic_mount() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1909,70 +1953,8 @@ _restic_prune() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") - flags+=("--tls-client-cert=") - two_word_flags+=("--tls-client-cert") - flags+=("--verbose") - flags+=("-v") - - must_have_one_flag=() - must_have_one_noun=() - noun_aliases=() -} - -_restic_rebuild-index() -{ - last_command="restic_rebuild-index" - - command_aliases=() - - commands=() - - flags=() - two_word_flags=() - local_nonpersistent_flags=() - flags_with_completion=() - flags_completion=() - - flags+=("--help") - flags+=("-h") - local_nonpersistent_flags+=("--help") - local_nonpersistent_flags+=("-h") - flags+=("--read-all-packs") - local_nonpersistent_flags+=("--read-all-packs") - flags+=("--cacert=") - two_word_flags+=("--cacert") - flags+=("--cache-dir=") - two_word_flags+=("--cache-dir") - flags+=("--cleanup-cache") - flags+=("--compression=") - two_word_flags+=("--compression") - flags+=("--insecure-tls") - flags+=("--json") - flags+=("--key-hint=") - two_word_flags+=("--key-hint") - flags+=("--limit-download=") - two_word_flags+=("--limit-download") - flags+=("--limit-upload=") - two_word_flags+=("--limit-upload") - flags+=("--no-cache") - flags+=("--no-lock") - flags+=("--option=") - two_word_flags+=("--option") - two_word_flags+=("-o") - flags+=("--pack-size=") - two_word_flags+=("--pack-size") - flags+=("--password-command=") - two_word_flags+=("--password-command") - flags+=("--password-file=") - two_word_flags+=("--password-file") - two_word_flags+=("-p") - flags+=("--quiet") - flags+=("-q") - flags+=("--repo=") - two_word_flags+=("--repo") - two_word_flags+=("-r") - flags+=("--repository-file=") - two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2035,6 +2017,351 @@ _restic_recover() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") + flags+=("--tls-client-cert=") + two_word_flags+=("--tls-client-cert") + flags+=("--verbose") + flags+=("-v") + + must_have_one_flag=() + must_have_one_noun=() + noun_aliases=() +} + +_restic_repair_help() +{ + last_command="restic_repair_help" + + command_aliases=() + + commands=() + + flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + + flags+=("--cacert=") + two_word_flags+=("--cacert") + flags+=("--cache-dir=") + two_word_flags+=("--cache-dir") + flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") + flags+=("--insecure-tls") + flags+=("--json") + flags+=("--key-hint=") + two_word_flags+=("--key-hint") + flags+=("--limit-download=") + two_word_flags+=("--limit-download") + flags+=("--limit-upload=") + two_word_flags+=("--limit-upload") + flags+=("--no-cache") + flags+=("--no-lock") + flags+=("--option=") + two_word_flags+=("--option") + two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") + flags+=("--password-command=") + two_word_flags+=("--password-command") + flags+=("--password-file=") + two_word_flags+=("--password-file") + two_word_flags+=("-p") + flags+=("--quiet") + flags+=("-q") + flags+=("--repo=") + two_word_flags+=("--repo") + two_word_flags+=("-r") + flags+=("--repository-file=") + two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") + flags+=("--tls-client-cert=") + two_word_flags+=("--tls-client-cert") + flags+=("--verbose") + flags+=("-v") + + must_have_one_flag=() + must_have_one_noun=() + has_completion_function=1 + noun_aliases=() +} + +_restic_repair_index() +{ + last_command="restic_repair_index" + + command_aliases=() + + commands=() + + flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + + flags+=("--help") + flags+=("-h") + local_nonpersistent_flags+=("--help") + local_nonpersistent_flags+=("-h") + flags+=("--read-all-packs") + local_nonpersistent_flags+=("--read-all-packs") + flags+=("--cacert=") + two_word_flags+=("--cacert") + flags+=("--cache-dir=") + two_word_flags+=("--cache-dir") + flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") + flags+=("--insecure-tls") + flags+=("--json") + flags+=("--key-hint=") + two_word_flags+=("--key-hint") + flags+=("--limit-download=") + two_word_flags+=("--limit-download") + flags+=("--limit-upload=") + two_word_flags+=("--limit-upload") + flags+=("--no-cache") + flags+=("--no-lock") + flags+=("--option=") + two_word_flags+=("--option") + two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") + flags+=("--password-command=") + two_word_flags+=("--password-command") + flags+=("--password-file=") + two_word_flags+=("--password-file") + two_word_flags+=("-p") + flags+=("--quiet") + flags+=("-q") + flags+=("--repo=") + two_word_flags+=("--repo") + two_word_flags+=("-r") + flags+=("--repository-file=") + two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") + flags+=("--tls-client-cert=") + two_word_flags+=("--tls-client-cert") + flags+=("--verbose") + flags+=("-v") + + must_have_one_flag=() + must_have_one_noun=() + noun_aliases=() +} + +_restic_repair_packs() +{ + last_command="restic_repair_packs" + + command_aliases=() + + commands=() + + flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + + flags+=("--help") + flags+=("-h") + local_nonpersistent_flags+=("--help") + local_nonpersistent_flags+=("-h") + flags+=("--cacert=") + two_word_flags+=("--cacert") + flags+=("--cache-dir=") + two_word_flags+=("--cache-dir") + flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") + flags+=("--insecure-tls") + flags+=("--json") + flags+=("--key-hint=") + two_word_flags+=("--key-hint") + flags+=("--limit-download=") + two_word_flags+=("--limit-download") + flags+=("--limit-upload=") + two_word_flags+=("--limit-upload") + flags+=("--no-cache") + flags+=("--no-lock") + flags+=("--option=") + two_word_flags+=("--option") + two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") + flags+=("--password-command=") + two_word_flags+=("--password-command") + flags+=("--password-file=") + two_word_flags+=("--password-file") + two_word_flags+=("-p") + flags+=("--quiet") + flags+=("-q") + flags+=("--repo=") + two_word_flags+=("--repo") + two_word_flags+=("-r") + flags+=("--repository-file=") + two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") + flags+=("--tls-client-cert=") + two_word_flags+=("--tls-client-cert") + flags+=("--verbose") + flags+=("-v") + + must_have_one_flag=() + must_have_one_noun=() + noun_aliases=() +} + +_restic_repair_snapshots() +{ + last_command="restic_repair_snapshots" + + command_aliases=() + + commands=() + + flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + + flags+=("--dry-run") + flags+=("-n") + local_nonpersistent_flags+=("--dry-run") + local_nonpersistent_flags+=("-n") + flags+=("--forget") + local_nonpersistent_flags+=("--forget") + flags+=("--help") + flags+=("-h") + local_nonpersistent_flags+=("--help") + local_nonpersistent_flags+=("-h") + flags+=("--host=") + two_word_flags+=("--host") + two_word_flags+=("-H") + local_nonpersistent_flags+=("--host") + local_nonpersistent_flags+=("--host=") + local_nonpersistent_flags+=("-H") + flags+=("--path=") + two_word_flags+=("--path") + local_nonpersistent_flags+=("--path") + local_nonpersistent_flags+=("--path=") + flags+=("--tag=") + two_word_flags+=("--tag") + local_nonpersistent_flags+=("--tag") + local_nonpersistent_flags+=("--tag=") + flags+=("--cacert=") + two_word_flags+=("--cacert") + flags+=("--cache-dir=") + two_word_flags+=("--cache-dir") + flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") + flags+=("--insecure-tls") + flags+=("--json") + flags+=("--key-hint=") + two_word_flags+=("--key-hint") + flags+=("--limit-download=") + two_word_flags+=("--limit-download") + flags+=("--limit-upload=") + two_word_flags+=("--limit-upload") + flags+=("--no-cache") + flags+=("--no-lock") + flags+=("--option=") + two_word_flags+=("--option") + two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") + flags+=("--password-command=") + two_word_flags+=("--password-command") + flags+=("--password-file=") + two_word_flags+=("--password-file") + two_word_flags+=("-p") + flags+=("--quiet") + flags+=("-q") + flags+=("--repo=") + two_word_flags+=("--repo") + two_word_flags+=("-r") + flags+=("--repository-file=") + two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") + flags+=("--tls-client-cert=") + two_word_flags+=("--tls-client-cert") + flags+=("--verbose") + flags+=("-v") + + must_have_one_flag=() + must_have_one_noun=() + noun_aliases=() +} + +_restic_repair() +{ + last_command="restic_repair" + + command_aliases=() + + commands=() + commands+=("help") + commands+=("index") + commands+=("packs") + commands+=("snapshots") + + flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + + flags+=("--help") + flags+=("-h") + local_nonpersistent_flags+=("--help") + local_nonpersistent_flags+=("-h") + flags+=("--cacert=") + two_word_flags+=("--cacert") + flags+=("--cache-dir=") + two_word_flags+=("--cache-dir") + flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") + flags+=("--insecure-tls") + flags+=("--json") + flags+=("--key-hint=") + two_word_flags+=("--key-hint") + flags+=("--limit-download=") + two_word_flags+=("--limit-download") + flags+=("--limit-upload=") + two_word_flags+=("--limit-upload") + flags+=("--no-cache") + flags+=("--no-lock") + flags+=("--option=") + two_word_flags+=("--option") + two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") + flags+=("--password-command=") + two_word_flags+=("--password-command") + flags+=("--password-file=") + two_word_flags+=("--password-file") + two_word_flags+=("-p") + flags+=("--quiet") + flags+=("-q") + flags+=("--repo=") + two_word_flags+=("--repo") + two_word_flags+=("-r") + flags+=("--repository-file=") + two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2141,6 +2468,8 @@ _restic_restore() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2241,6 +2570,8 @@ _restic_rewrite() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2307,6 +2638,8 @@ _restic_self-update() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2397,6 +2730,8 @@ _restic_snapshots() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2477,6 +2812,8 @@ _restic_stats() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2565,6 +2902,8 @@ _restic_tag() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2629,6 +2968,8 @@ _restic_unlock() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2691,6 +3032,8 @@ _restic_version() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2726,8 +3069,8 @@ _restic_root_command() commands+=("migrate") commands+=("mount") commands+=("prune") - commands+=("rebuild-index") commands+=("recover") + commands+=("repair") commands+=("restore") commands+=("rewrite") commands+=("self-update") @@ -2781,6 +3124,8 @@ _restic_root_command() two_word_flags+=("-r") flags+=("--repository-file=") two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") diff --git a/doc/conf.py b/doc/conf.py index 3fd8dc119..f40a8c79e 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -12,14 +12,16 @@ # # All configuration values have a default; values that are commented out # serve to show the default. -import os # -- General configuration ------------------------------------------------ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.extlinks'] +extensions = [ + 'sphinx.ext.extlinks', + 'sphinx_rtd_theme', +] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -35,7 +37,7 @@ master_doc = 'index' # General information about the project. project = 'restic' -copyright = '2018, restic authors' +copyright = '2023, restic authors' author = 'fd0' # The version info for the project you're documenting, acts as replacement for @@ -54,7 +56,7 @@ release = version # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -72,21 +74,11 @@ todo_include_todos = False # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -# -if os.environ.get('READTHEDOCS') == 'True': - html_context = { - 'css_files': [ - 'https://media.readthedocs.org/css/sphinx_rtd_theme.css', - 'https://media.readthedocs.org/css/readthedocs-doc-embed.css', - '_static/css/restic.css', - ] - } -else: - # we're not built by rtd => add rtd-theme - import sphinx_rtd_theme - html_theme = 'sphinx_rtd_theme' - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - html_style = 'css/restic.css' +html_theme = 'sphinx_rtd_theme' + +html_css_files = [ + 'css/restic.css', +] html_logo = 'logo/logo.png' diff --git a/doc/design.rst b/doc/design.rst index 94dabdc34..1e00a3358 100644 --- a/doc/design.rst +++ b/doc/design.rst @@ -48,7 +48,7 @@ be used instead of the complete filename. Apart from the files stored within the ``keys`` and ``data`` directories, all files are encrypted with AES-256 in counter mode (CTR). The integrity of the encrypted data is secured by a Poly1305-AES message authentication -code (sometimes also referred to as a "signature"). +code (MAC). Files in the ``data`` directory ("pack files") consist of multiple parts which are all independently encrypted and authenticated, see below. @@ -501,7 +501,7 @@ the JSON is indented): } A tree contains a list of entries (in the field ``nodes``) which contain -meta data like a name and timestamps. Note that there are some specialities of how +meta data like a name and timestamps. Note that there are some specialties of how this metadata is generated: - The name is quoted using `strconv.Quote `__ @@ -519,7 +519,7 @@ to print a tree. The tree referenced above can be dumped as follows: .. code-block:: console - $ restic -r /tmp/restic-repo cat blob b26e315b0988ddcd1cee64c351d13a100fedbc9fdbb144a67d1b765ab280b4dc + $ restic -r /tmp/restic-repo cat blob b26e315b0988ddcd1cee64c351d13a100fedbc9fdbb144a67d1b765ab280b4dc | jq . enter password for repository: { "nodes": [ @@ -548,6 +548,39 @@ This tree contains a file entry. This time, the ``subtree`` field is not present and the ``content`` field contains a list with one plain text SHA-256 hash. +A symlink uses the following data structure: + +.. code-block:: console + + $ restic -r /tmp/restic-repo cat blob 4c0a7d500bd1482ba01752e77c8d5a923304777d96b6522fae7c11e99b4e6fa6 | jq . + enter password for repository: + { + "nodes": [ + { + "name": "testlink", + "type": "symlink", + "mode": 134218239, + "mtime": "2023-07-25T20:01:44.007465374+02:00", + "atime": "2023-07-25T20:01:44.007465374+02:00", + "ctime": "2023-07-25T20:01:44.007465374+02:00", + "uid": 1000, + "gid": 100, + "user": "fd0", + "inode": 33734827, + "links": 1, + "linktarget": "example_target", + "content": null + }, + [...] + ] + } + +The symlink target is stored in the field `linktarget`. As JSON strings can +only contain valid unicode, an exception applies if the `linktarget` is not a +valid UTF-8 string. Since restic 0.16.0, in such a case the `linktarget_raw` +field contains a base64 encoded version of the raw linktarget. The +`linktarget_raw` field is only set if `linktarget` cannot be encoded correctly. + The command ``restic cat blob`` can also be used to extract and decrypt data given a plaintext ID, e.g. for the data mentioned above: diff --git a/doc/faq.rst b/doc/faq.rst index 8e56b5d9e..e8ef2de5e 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -74,7 +74,7 @@ $ restic backup --exclude "~/documents" ~ This command will result in a complete backup of the current logged in user's home directory and it won't exclude the folder ``~/documents/`` - which is not what the user wanted to achieve. The problem is how the path to ``~/documents`` is passed to restic. -In order to spot an issue like this, you can make use of the following ruby command preceding your restic command. +In order to spot an issue like this, you can make use of the following ruby command preceeding your restic command. :: diff --git a/doc/fish-completion.fish b/doc/fish-completion.fish index aa60d536d..7db10cb20 100644 --- a/doc/fish-completion.fish +++ b/doc/fish-completion.fish @@ -55,6 +55,60 @@ function __restic_perform_completion printf "%s\n" "$directiveLine" end +# this function limits calls to __restic_perform_completion, by caching the result behind $__restic_perform_completion_once_result +function __restic_perform_completion_once + __restic_debug "Starting __restic_perform_completion_once" + + if test -n "$__restic_perform_completion_once_result" + __restic_debug "Seems like a valid result already exists, skipping __restic_perform_completion" + return 0 + end + + set --global __restic_perform_completion_once_result (__restic_perform_completion) + if test -z "$__restic_perform_completion_once_result" + __restic_debug "No completions, probably due to a failure" + return 1 + end + + __restic_debug "Performed completions and set __restic_perform_completion_once_result" + return 0 +end + +# this function is used to clear the $__restic_perform_completion_once_result variable after completions are run +function __restic_clear_perform_completion_once_result + __restic_debug "" + __restic_debug "========= clearing previously set __restic_perform_completion_once_result variable ==========" + set --erase __restic_perform_completion_once_result + __restic_debug "Successfully erased the variable __restic_perform_completion_once_result" +end + +function __restic_requires_order_preservation + __restic_debug "" + __restic_debug "========= checking if order preservation is required ==========" + + __restic_perform_completion_once + if test -z "$__restic_perform_completion_once_result" + __restic_debug "Error determining if order preservation is required" + return 1 + end + + set -l directive (string sub --start 2 $__restic_perform_completion_once_result[-1]) + __restic_debug "Directive is: $directive" + + set -l shellCompDirectiveKeepOrder 32 + set -l keeporder (math (math --scale 0 $directive / $shellCompDirectiveKeepOrder) % 2) + __restic_debug "Keeporder is: $keeporder" + + if test $keeporder -ne 0 + __restic_debug "This does require order preservation" + return 0 + end + + __restic_debug "This doesn't require order preservation" + return 1 +end + + # This function does two things: # - Obtain the completions and store them in the global __restic_comp_results # - Return false if file completion should be performed @@ -65,17 +119,17 @@ function __restic_prepare_completions # Start fresh set --erase __restic_comp_results - set -l results (__restic_perform_completion) - __restic_debug "Completion results: $results" + __restic_perform_completion_once + __restic_debug "Completion results: $__restic_perform_completion_once_result" - if test -z "$results" + if test -z "$__restic_perform_completion_once_result" __restic_debug "No completion, probably due to a failure" # Might as well do file completion, in case it helps return 1 end - set -l directive (string sub --start 2 $results[-1]) - set --global __restic_comp_results $results[1..-2] + set -l directive (string sub --start 2 $__restic_perform_completion_once_result[-1]) + set --global __restic_comp_results $__restic_perform_completion_once_result[1..-2] __restic_debug "Completions are: $__restic_comp_results" __restic_debug "Directive is: $directive" @@ -171,7 +225,11 @@ end # Remove any pre-existing completions for the program since we will be handling all of them. complete -c restic -e +# this will get called after the two calls below and clear the $__restic_perform_completion_once_result global +complete -c restic -n '__restic_clear_perform_completion_once_result' # The call to __restic_prepare_completions will setup __restic_comp_results # which provides the program's completion choices. -complete -c restic -n '__restic_prepare_completions' -f -a '$__restic_comp_results' - +# If this doesn't require order preservation, we don't use the -k flag +complete -c restic -n 'not __restic_requires_order_preservation && __restic_prepare_completions' -f -a '$__restic_comp_results' +# otherwise we use the -k flag +complete -k -c restic -n '__restic_requires_order_preservation && __restic_prepare_completions' -f -a '$__restic_comp_results' diff --git a/doc/man/restic-backup.1 b/doc/man/restic-backup.1 index 4297c3b8e..c3bccdfa5 100644 --- a/doc/man/restic-backup.1 +++ b/doc/man/restic-backup.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -31,7 +31,7 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea .PP \fB-e\fP, \fB--exclude\fP=[] - exclude a \fB\fCpattern\fR (can be specified multiple times) + exclude a \fBpattern\fR (can be specified multiple times) .PP \fB--exclude-caches\fP[=false] @@ -39,47 +39,51 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea .PP \fB--exclude-file\fP=[] - read exclude patterns from a \fB\fCfile\fR (can be specified multiple times) + read exclude patterns from a \fBfile\fR (can be specified multiple times) .PP \fB--exclude-if-present\fP=[] - takes \fB\fCfilename[:header]\fR, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times) + takes \fBfilename[:header]\fR, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times) .PP \fB--exclude-larger-than\fP="" - max \fB\fCsize\fR of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T) + max \fBsize\fR of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T) .PP \fB--files-from\fP=[] - read the files to backup from \fB\fCfile\fR (can be combined with file args; can be specified multiple times) + read the files to backup from \fBfile\fR (can be combined with file args; can be specified multiple times) .PP \fB--files-from-raw\fP=[] - read the files to backup from \fB\fCfile\fR (can be combined with file args; can be specified multiple times) + read the files to backup from \fBfile\fR (can be combined with file args; can be specified multiple times) .PP \fB--files-from-verbatim\fP=[] - read the files to backup from \fB\fCfile\fR (can be combined with file args; can be specified multiple times) + read the files to backup from \fBfile\fR (can be combined with file args; can be specified multiple times) .PP \fB-f\fP, \fB--force\fP[=false] force re-reading the target files/directories (overrides the "parent" flag) +.PP +\fB-g\fP, \fB--group-by\fP=host,paths + \fBgroup\fR snapshots by host, paths and/or tags, separated by comma (disable grouping with '') + .PP \fB-h\fP, \fB--help\fP[=false] help for backup .PP \fB-H\fP, \fB--host\fP="" - set the \fB\fChostname\fR for the snapshot manually. To prevent an expensive rescan use the "parent" flag + set the \fBhostname\fR for the snapshot manually. To prevent an expensive rescan use the "parent" flag .PP \fB--iexclude\fP=[] - same as --exclude \fB\fCpattern\fR but ignores the casing of filenames + same as --exclude \fBpattern\fR but ignores the casing of filenames .PP \fB--iexclude-file\fP=[] - same as --exclude-file but ignores casing of \fB\fCfile\fRnames in patterns + same as --exclude-file but ignores casing of \fBfile\fRnames in patterns .PP \fB--ignore-ctime\fP[=false] @@ -99,11 +103,11 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea .PP \fB--parent\fP="" - use this parent \fB\fCsnapshot\fR (default: last snapshot in the repository that has the same target files/directories, and is not newer than the snapshot time) + use this parent \fBsnapshot\fR (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time) .PP \fB--read-concurrency\fP=0 - read \fB\fCn\fR files concurrently (default: $RESTIC_READ_CONCURRENCY or 2) + read \fBn\fR files concurrently (default: $RESTIC_READ_CONCURRENCY or 2) .PP \fB--stdin\fP[=false] @@ -111,15 +115,15 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea .PP \fB--stdin-filename\fP="stdin" - \fB\fCfilename\fR to use when reading from stdin + \fBfilename\fR to use when reading from stdin .PP \fB--tag\fP=[] - add \fB\fCtags\fR for the new snapshot in the format \fB\fCtag[,tag,...]\fR (can be specified multiple times) + add \fBtags\fR for the new snapshot in the format \fBtag[,tag,...]\fR (can be specified multiple times) .PP \fB--time\fP="" - \fB\fCtime\fR of the backup (ex. '2012-11-01 22:08:41') (default: now) + \fBtime\fR of the backup (ex. '2012-11-01 22:08:41') (default: now) .PP \fB--with-atime\fP[=false] @@ -129,11 +133,11 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -141,7 +145,7 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -153,15 +157,15 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -173,19 +177,19 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -193,15 +197,19 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-cache.1 b/doc/man/restic-cache.1 index 3552fb1dc..3ae27ea57 100644 --- a/doc/man/restic-cache.1 +++ b/doc/man/restic-cache.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -32,7 +32,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--max-age\fP=30 - max age in \fB\fCdays\fR for cache directories to be considered old + max age in \fBdays\fR for cache directories to be considered old .PP \fB--no-size\fP[=false] @@ -42,11 +42,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -54,7 +54,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -66,15 +66,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -86,19 +86,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -106,15 +106,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-cat.1 b/doc/man/restic-cat.1 index 2e787fa06..c1df138aa 100644 --- a/doc/man/restic-cat.1 +++ b/doc/man/restic-cat.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -8,7 +8,7 @@ restic-cat - Print internal objects to stdout .SH SYNOPSIS .PP -\fBrestic cat [flags] [pack|blob|snapshot|index|key|masterkey|config|lock] ID\fP +\fBrestic cat [flags] [masterkey|config|pack ID|blob ID|snapshot ID|index ID|key ID|lock ID|tree snapshot:subfolder]\fP .SH DESCRIPTION @@ -30,11 +30,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -42,7 +42,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -54,15 +54,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -74,19 +74,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -94,15 +94,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-check.1 b/doc/man/restic-check.1 index e641fc2b5..17eb972bc 100644 --- a/doc/man/restic-check.1 +++ b/doc/man/restic-check.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -37,21 +37,21 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--read-data-subset\fP="" - read a \fB\fCsubset\fR of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset + read a \fBsubset\fR of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset .PP \fB--with-cache\fP[=false] - use the cache + use existing cache, only read uncached data from repository .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -59,7 +59,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -71,15 +71,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -91,19 +91,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -111,15 +111,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-copy.1 b/doc/man/restic-copy.1 index 53badecc9..be8f21e25 100644 --- a/doc/man/restic-copy.1 +++ b/doc/man/restic-copy.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -37,19 +37,19 @@ new destination repository using the "init" command. .PP \fB--from-password-command\fP="" - shell \fB\fCcommand\fR to obtain the source repository password from (default: $RESTIC_FROM_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the source repository password from (default: $RESTIC_FROM_PASSWORD_COMMAND) .PP \fB--from-password-file\fP="" - \fB\fCfile\fR to read the source repository password from (default: $RESTIC_FROM_PASSWORD_FILE) + \fBfile\fR to read the source repository password from (default: $RESTIC_FROM_PASSWORD_FILE) .PP \fB--from-repo\fP="" - source \fB\fCrepository\fR to copy snapshots from (default: $RESTIC_FROM_REPOSITORY) + source \fBrepository\fR to copy snapshots from (default: $RESTIC_FROM_REPOSITORY) .PP \fB--from-repository-file\fP="" - \fB\fCfile\fR from which to read the source repository location to copy snapshots from (default: $RESTIC_FROM_REPOSITORY_FILE) + \fBfile\fR from which to read the source repository location to copy snapshots from (default: $RESTIC_FROM_REPOSITORY_FILE) .PP \fB-h\fP, \fB--help\fP[=false] @@ -57,25 +57,25 @@ new destination repository using the "init" command. .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fB\fChost\fR (can be specified multiple times) + only consider snapshots for this \fBhost\fR (can be specified multiple times) .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fB\fCpath\fR (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times) .PP \fB--tag\fP=[] - only consider snapshots including \fB\fCtag[,tag,...]\fR (can be specified multiple times) + only consider snapshots including \fBtag[,tag,...]\fR (can be specified multiple times) .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -83,7 +83,7 @@ new destination repository using the "init" command. .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -95,15 +95,15 @@ new destination repository using the "init" command. .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -115,19 +115,19 @@ new destination repository using the "init" command. .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -135,15 +135,19 @@ new destination repository using the "init" command. .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-diff.1 b/doc/man/restic-diff.1 index 31c34dc8a..a01a2562b 100644 --- a/doc/man/restic-diff.1 +++ b/doc/man/restic-diff.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -8,7 +8,7 @@ restic-diff - Show differences between two snapshots .SH SYNOPSIS .PP -\fBrestic diff [flags] snapshot-ID snapshot-ID\fP +\fBrestic diff [flags] snapshotID snapshotID\fP .SH DESCRIPTION @@ -31,6 +31,11 @@ T The type was changed, e.g. a file was made a symlink .RE +.PP +To only compare files in specific subfolders, you can use the +":" syntax, where "subfolder" is a path within the +snapshot. + .SH EXIT STATUS .PP @@ -50,11 +55,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -62,7 +67,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -74,15 +79,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -94,19 +99,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -114,15 +119,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-dump.1 b/doc/man/restic-dump.1 index 61b3b3ec8..6fa1f8200 100644 --- a/doc/man/restic-dump.1 +++ b/doc/man/restic-dump.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -19,9 +19,14 @@ as a tar (default) or zip file containing the contents of the specified folder. Pass "/" as file name to dump the whole snapshot as an archive file. .PP -The special snapshot "latest" can be used to use the latest snapshot in the +The special snapshotID "latest" can be used to use the latest snapshot in the repository. +.PP +To include the folder content at the root of the archive, you can use the +":" syntax, where "subfolder" is a path within the +snapshot. + .SH EXIT STATUS .PP @@ -31,7 +36,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .SH OPTIONS .PP \fB-a\fP, \fB--archive\fP="tar" - set archive \fB\fCformat\fR as "tar" or "zip" + set archive \fBformat\fR as "tar" or "zip" .PP \fB-h\fP, \fB--help\fP[=false] @@ -39,25 +44,25 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fB\fChost\fR, when snapshot ID "latest" is given (can be specified multiple times) + only consider snapshots for this \fBhost\fR, when snapshot ID "latest" is given (can be specified multiple times) .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fB\fCpath\fR, when snapshot ID "latest" is given (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR, when snapshot ID "latest" is given (can be specified multiple times) .PP \fB--tag\fP=[] - only consider snapshots including \fB\fCtag[,tag,...]\fR, when snapshot ID "latest" is given (can be specified multiple times) + only consider snapshots including \fBtag[,tag,...]\fR, when snapshot ID "latest" is given (can be specified multiple times) .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -65,7 +70,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -77,15 +82,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -97,19 +102,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -117,15 +122,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-find.1 b/doc/man/restic-find.1 index 9fa4dd811..72bc3a0b6 100644 --- a/doc/man/restic-find.1 +++ b/doc/man/restic-find.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -29,7 +29,11 @@ It can also be used to search for restic blobs or trees for troubleshooting. .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fB\fChost\fR (can be specified multiple times) + only consider snapshots for this \fBhost\fR (can be specified multiple times) + +.PP +\fB--human-readable\fP[=false] + print sizes in human readable format .PP \fB-i\fP, \fB--ignore-case\fP[=false] @@ -53,7 +57,7 @@ It can also be used to search for restic blobs or trees for troubleshooting. .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fB\fCpath\fR (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times) .PP \fB--show-pack-id\fP[=false] @@ -61,11 +65,11 @@ It can also be used to search for restic blobs or trees for troubleshooting. .PP \fB-s\fP, \fB--snapshot\fP=[] - snapshot \fB\fCid\fR to search in (can be given multiple times) + snapshot \fBid\fR to search in (can be given multiple times) .PP \fB--tag\fP=[] - only consider snapshots including \fB\fCtag[,tag,...]\fR (can be specified multiple times) + only consider snapshots including \fBtag[,tag,...]\fR (can be specified multiple times) .PP \fB--tree\fP[=false] @@ -75,11 +79,11 @@ It can also be used to search for restic blobs or trees for troubleshooting. .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -87,7 +91,7 @@ It can also be used to search for restic blobs or trees for troubleshooting. .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -99,15 +103,15 @@ It can also be used to search for restic blobs or trees for troubleshooting. .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -119,19 +123,19 @@ It can also be used to search for restic blobs or trees for troubleshooting. .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -139,15 +143,19 @@ It can also be used to search for restic blobs or trees for troubleshooting. .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] @@ -155,10 +163,7 @@ It can also be used to search for restic blobs or trees for troubleshooting. .SH EXAMPLE -.PP -.RS - -.nf +.EX restic find config.json restic find --json "*.yml" "*.json" restic find --json --blob 420f620f b46ebe8a ddd38656 @@ -172,8 +177,7 @@ EXIT STATUS Exit status is 0 if the command was successful, and non-zero if there was any error. -.fi -.RE +.EE .SH SEE ALSO diff --git a/doc/man/restic-forget.1 b/doc/man/restic-forget.1 index d8a69856e..757022a21 100644 --- a/doc/man/restic-forget.1 +++ b/doc/man/restic-forget.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -35,75 +35,75 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .SH OPTIONS .PP \fB-l\fP, \fB--keep-last\fP=0 - keep the last \fB\fCn\fR snapshots + keep the last \fBn\fR snapshots (use 'unlimited' to keep all snapshots) .PP \fB-H\fP, \fB--keep-hourly\fP=0 - keep the last \fB\fCn\fR hourly snapshots + keep the last \fBn\fR hourly snapshots (use 'unlimited' to keep all hourly snapshots) .PP \fB-d\fP, \fB--keep-daily\fP=0 - keep the last \fB\fCn\fR daily snapshots + keep the last \fBn\fR daily snapshots (use 'unlimited' to keep all daily snapshots) .PP \fB-w\fP, \fB--keep-weekly\fP=0 - keep the last \fB\fCn\fR weekly snapshots + keep the last \fBn\fR weekly snapshots (use 'unlimited' to keep all weekly snapshots) .PP \fB-m\fP, \fB--keep-monthly\fP=0 - keep the last \fB\fCn\fR monthly snapshots + keep the last \fBn\fR monthly snapshots (use 'unlimited' to keep all monthly snapshots) .PP \fB-y\fP, \fB--keep-yearly\fP=0 - keep the last \fB\fCn\fR yearly snapshots + keep the last \fBn\fR yearly snapshots (use 'unlimited' to keep all yearly snapshots) .PP \fB--keep-within\fP= - keep snapshots that are newer than \fB\fCduration\fR (eg. 1y5m7d2h) relative to the latest snapshot + keep snapshots that are newer than \fBduration\fR (eg. 1y5m7d2h) relative to the latest snapshot .PP \fB--keep-within-hourly\fP= - keep hourly snapshots that are newer than \fB\fCduration\fR (eg. 1y5m7d2h) relative to the latest snapshot + keep hourly snapshots that are newer than \fBduration\fR (eg. 1y5m7d2h) relative to the latest snapshot .PP \fB--keep-within-daily\fP= - keep daily snapshots that are newer than \fB\fCduration\fR (eg. 1y5m7d2h) relative to the latest snapshot + keep daily snapshots that are newer than \fBduration\fR (eg. 1y5m7d2h) relative to the latest snapshot .PP \fB--keep-within-weekly\fP= - keep weekly snapshots that are newer than \fB\fCduration\fR (eg. 1y5m7d2h) relative to the latest snapshot + keep weekly snapshots that are newer than \fBduration\fR (eg. 1y5m7d2h) relative to the latest snapshot .PP \fB--keep-within-monthly\fP= - keep monthly snapshots that are newer than \fB\fCduration\fR (eg. 1y5m7d2h) relative to the latest snapshot + keep monthly snapshots that are newer than \fBduration\fR (eg. 1y5m7d2h) relative to the latest snapshot .PP \fB--keep-within-yearly\fP= - keep yearly snapshots that are newer than \fB\fCduration\fR (eg. 1y5m7d2h) relative to the latest snapshot + keep yearly snapshots that are newer than \fBduration\fR (eg. 1y5m7d2h) relative to the latest snapshot .PP \fB--keep-tag\fP=[] - keep snapshots with this \fB\fCtaglist\fR (can be specified multiple times) + keep snapshots with this \fBtaglist\fR (can be specified multiple times) .PP \fB--host\fP=[] - only consider snapshots for this \fB\fChost\fR (can be specified multiple times) + only consider snapshots for this \fBhost\fR (can be specified multiple times) .PP \fB--tag\fP=[] - only consider snapshots including \fB\fCtag[,tag,...]\fR (can be specified multiple times) + only consider snapshots including \fBtag[,tag,...]\fR (can be specified multiple times) .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fB\fCpath\fR (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times) .PP \fB-c\fP, \fB--compact\fP[=false] use compact output format .PP -\fB-g\fP, \fB--group-by\fP="host,paths" - \fB\fCgroup\fR snapshots by host, paths and/or tags, separated by comma (disable grouping with '') +\fB-g\fP, \fB--group-by\fP=host,paths + \fBgroup\fR snapshots by host, paths and/or tags, separated by comma (disable grouping with '') .PP \fB-n\fP, \fB--dry-run\fP[=false] @@ -115,11 +115,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--max-unused\fP="5%" - tolerate given \fB\fClimit\fR of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited') + tolerate given \fBlimit\fR of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited') .PP \fB--max-repack-size\fP="" - maximum \fB\fCsize\fR to repack (allowed suffixes: k/K, m/M, g/G, t/T) + maximum \fBsize\fR to repack (allowed suffixes: k/K, m/M, g/G, t/T) .PP \fB--repack-cacheable-only\fP[=false] @@ -141,11 +141,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -153,7 +153,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -165,15 +165,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -185,19 +185,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -205,15 +205,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-generate.1 b/doc/man/restic-generate.1 index 6b54ebfca..aef3a5e55 100644 --- a/doc/man/restic-generate.1 +++ b/doc/man/restic-generate.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -25,11 +25,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .SH OPTIONS .PP \fB--bash-completion\fP="" - write bash completion \fB\fCfile\fR + write bash completion \fBfile\fR .PP \fB--fish-completion\fP="" - write fish completion \fB\fCfile\fR + write fish completion \fBfile\fR .PP \fB-h\fP, \fB--help\fP[=false] @@ -37,25 +37,25 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--man\fP="" - write man pages to \fB\fCdirectory\fR + write man pages to \fBdirectory\fR .PP \fB--powershell-completion\fP="" - write powershell completion \fB\fCfile\fR + write powershell completion \fBfile\fR .PP \fB--zsh-completion\fP="" - write zsh completion \fB\fCfile\fR + write zsh completion \fBfile\fR .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -63,7 +63,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -75,15 +75,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -95,19 +95,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -115,15 +115,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-init.1 b/doc/man/restic-init.1 index 194f31756..27d7f5874 100644 --- a/doc/man/restic-init.1 +++ b/doc/man/restic-init.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -32,19 +32,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--from-password-command\fP="" - shell \fB\fCcommand\fR to obtain the source repository password from (default: $RESTIC_FROM_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the source repository password from (default: $RESTIC_FROM_PASSWORD_COMMAND) .PP \fB--from-password-file\fP="" - \fB\fCfile\fR to read the source repository password from (default: $RESTIC_FROM_PASSWORD_FILE) + \fBfile\fR to read the source repository password from (default: $RESTIC_FROM_PASSWORD_FILE) .PP \fB--from-repo\fP="" - source \fB\fCrepository\fR to copy chunker parameters from (default: $RESTIC_FROM_REPOSITORY) + source \fBrepository\fR to copy chunker parameters from (default: $RESTIC_FROM_REPOSITORY) .PP \fB--from-repository-file\fP="" - \fB\fCfile\fR from which to read the source repository location to copy chunker parameters from (default: $RESTIC_FROM_REPOSITORY_FILE) + \fBfile\fR from which to read the source repository location to copy chunker parameters from (default: $RESTIC_FROM_REPOSITORY_FILE) .PP \fB-h\fP, \fB--help\fP[=false] @@ -58,11 +58,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -70,7 +70,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -82,15 +82,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -102,19 +102,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -122,15 +122,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-key.1 b/doc/man/restic-key.1 index 4163cefa5..855ef5443 100644 --- a/doc/man/restic-key.1 +++ b/doc/man/restic-key.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -32,7 +32,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--new-password-file\fP="" - \fB\fCfile\fR from which to read the new password + \fBfile\fR from which to read the new password .PP \fB--user\fP="" @@ -42,11 +42,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -54,7 +54,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -66,15 +66,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -86,19 +86,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -106,15 +106,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-list.1 b/doc/man/restic-list.1 index 6683e2c47..95eeac5f7 100644 --- a/doc/man/restic-list.1 +++ b/doc/man/restic-list.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -30,11 +30,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -42,7 +42,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -54,15 +54,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -74,19 +74,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -94,15 +94,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-ls.1 b/doc/man/restic-ls.1 index a16716434..0cd0f5a88 100644 --- a/doc/man/restic-ls.1 +++ b/doc/man/restic-ls.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -43,7 +43,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fB\fChost\fR, when snapshot ID "latest" is given (can be specified multiple times) + only consider snapshots for this \fBhost\fR, when snapshot ID "latest" is given (can be specified multiple times) + +.PP +\fB--human-readable\fP[=false] + print sizes in human readable format .PP \fB-l\fP, \fB--long\fP[=false] @@ -51,7 +55,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fB\fCpath\fR, when snapshot ID "latest" is given (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR, when snapshot ID "latest" is given (can be specified multiple times) .PP \fB--recursive\fP[=false] @@ -59,17 +63,17 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--tag\fP=[] - only consider snapshots including \fB\fCtag[,tag,...]\fR, when snapshot ID "latest" is given (can be specified multiple times) + only consider snapshots including \fBtag[,tag,...]\fR, when snapshot ID "latest" is given (can be specified multiple times) .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -77,7 +81,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -89,15 +93,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -109,19 +113,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -129,15 +133,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-migrate.1 b/doc/man/restic-migrate.1 index d8127090e..eca0ef8e1 100644 --- a/doc/man/restic-migrate.1 +++ b/doc/man/restic-migrate.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -36,11 +36,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -48,7 +48,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -60,15 +60,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -80,19 +80,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -100,15 +100,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-mount.1 b/doc/man/restic-mount.1 index ce4f893a7..33c016ffa 100644 --- a/doc/man/restic-mount.1 +++ b/doc/man/restic-mount.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -26,26 +26,18 @@ you can pass a time template via --time-template and path templates via .PP Example time template without colons: -.PP -.RS - -.nf +.EX --time-template "2006-01-02_15-04-05" -.fi -.RE +.EE .PP You need to specify a sample format for exactly the following timestamp: -.PP -.RS - -.nf +.EX Mon Jan 2 15:04:05 -0700 MST 2006 -.fi -.RE +.EE .PP For details please see the documentation for time.Format() at: @@ -84,7 +76,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fB\fChost\fR (can be specified multiple times) + only consider snapshots for this \fBhost\fR (can be specified multiple times) .PP \fB--no-default-permissions\fP[=false] @@ -96,29 +88,29 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fB\fCpath\fR (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times) .PP \fB--path-template\fP=[] - set \fB\fCtemplate\fR for path names (can be specified multiple times) + set \fBtemplate\fR for path names (can be specified multiple times) .PP \fB--tag\fP=[] - only consider snapshots including \fB\fCtag[,tag,...]\fR (can be specified multiple times) + only consider snapshots including \fBtag[,tag,...]\fR (can be specified multiple times) .PP \fB--time-template\fP="2006-01-02T15:04:05Z07:00" - set \fB\fCtemplate\fR to use for times + set \fBtemplate\fR to use for times .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -126,7 +118,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -138,15 +130,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -158,19 +150,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -178,15 +170,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-prune.1 b/doc/man/restic-prune.1 index 197cb1130..e4a32cac3 100644 --- a/doc/man/restic-prune.1 +++ b/doc/man/restic-prune.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -33,11 +33,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--max-repack-size\fP="" - maximum \fB\fCsize\fR to repack (allowed suffixes: k/K, m/M, g/G, t/T) + maximum \fBsize\fR to repack (allowed suffixes: k/K, m/M, g/G, t/T) .PP \fB--max-unused\fP="5%" - tolerate given \fB\fClimit\fR of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited') + tolerate given \fBlimit\fR of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited') .PP \fB--repack-cacheable-only\fP[=false] @@ -59,11 +59,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -71,7 +71,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -83,15 +83,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -103,19 +103,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -123,15 +123,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-rebuild-index.1 b/doc/man/restic-rebuild-index.1 deleted file mode 100644 index 18878b66f..000000000 --- a/doc/man/restic-rebuild-index.1 +++ /dev/null @@ -1,119 +0,0 @@ -.nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" - -.SH NAME -.PP -restic-rebuild-index - Build a new index - - -.SH SYNOPSIS -.PP -\fBrestic rebuild-index [flags]\fP - - -.SH DESCRIPTION -.PP -The "rebuild-index" command creates a new index based on the pack files in the -repository. - - -.SH EXIT STATUS -.PP -Exit status is 0 if the command was successful, and non-zero if there was any error. - - -.SH OPTIONS -.PP -\fB-h\fP, \fB--help\fP[=false] - help for rebuild-index - -.PP -\fB--read-all-packs\fP[=false] - read all pack files to generate new index from scratch - - -.SH OPTIONS INHERITED FROM PARENT COMMANDS -.PP -\fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) - -.PP -\fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) - -.PP -\fB--cleanup-cache\fP[=false] - auto remove old cache directories - -.PP -\fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) - -.PP -\fB--insecure-tls\fP[=false] - skip TLS certificate verification when connecting to the repository (insecure) - -.PP -\fB--json\fP[=false] - set output mode to JSON for commands that support it - -.PP -\fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) - -.PP -\fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) - -.PP -\fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) - -.PP -\fB--no-cache\fP[=false] - do not use a local cache - -.PP -\fB--no-lock\fP[=false] - do not lock the repository, this allows some operations on read-only repositories - -.PP -\fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) - -.PP -\fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) - -.PP -\fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) - -.PP -\fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) - -.PP -\fB-q\fP, \fB--quiet\fP[=false] - do not output comprehensive progress report - -.PP -\fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) - -.PP -\fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) - -.PP -\fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key - -.PP -\fB-v\fP, \fB--verbose\fP[=0] - be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) - - -.SH SEE ALSO -.PP -\fBrestic(1)\fP diff --git a/doc/man/restic-recover.1 b/doc/man/restic-recover.1 index aa3441156..26d2fc7bd 100644 --- a/doc/man/restic-recover.1 +++ b/doc/man/restic-recover.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -32,11 +32,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -44,7 +44,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -56,15 +56,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -76,19 +76,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -96,15 +96,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-repair-index.1 b/doc/man/restic-repair-index.1 new file mode 100644 index 000000000..35e2845b8 --- /dev/null +++ b/doc/man/restic-repair-index.1 @@ -0,0 +1,123 @@ +.nh +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" + +.SH NAME +.PP +restic-repair-index - Build a new index + + +.SH SYNOPSIS +.PP +\fBrestic repair index [flags]\fP + + +.SH DESCRIPTION +.PP +The "repair index" command creates a new index based on the pack files in the +repository. + + +.SH EXIT STATUS +.PP +Exit status is 0 if the command was successful, and non-zero if there was any error. + + +.SH OPTIONS +.PP +\fB-h\fP, \fB--help\fP[=false] + help for index + +.PP +\fB--read-all-packs\fP[=false] + read all pack files to generate new index from scratch + + +.SH OPTIONS INHERITED FROM PARENT COMMANDS +.PP +\fB--cacert\fP=[] + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) + +.PP +\fB--cache-dir\fP="" + set the cache \fBdirectory\fR\&. (default: use system default cache directory) + +.PP +\fB--cleanup-cache\fP[=false] + auto remove old cache directories + +.PP +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) + +.PP +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] + set output mode to JSON for commands that support it + +.PP +\fB--key-hint\fP="" + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + +.PP +\fB--limit-download\fP=0 + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--limit-upload\fP=0 + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--no-cache\fP[=false] + do not use a local cache + +.PP +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories + +.PP +\fB-o\fP, \fB--option\fP=[] + set extended option (\fBkey=value\fR, can be specified multiple times) + +.PP +\fB--pack-size\fP=0 + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + +.PP +\fB-p\fP, \fB--password-file\fP="" + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + +.PP +\fB-q\fP, \fB--quiet\fP[=false] + do not output comprehensive progress report + +.PP +\fB-r\fP, \fB--repo\fP="" + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + +.PP +\fB--repository-file\fP="" + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) + +.PP +\fB--tls-client-cert\fP="" + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) + +.PP +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) + + +.SH SEE ALSO +.PP +\fBrestic-repair(1)\fP diff --git a/doc/man/restic-repair-packs.1 b/doc/man/restic-repair-packs.1 new file mode 100644 index 000000000..b21211925 --- /dev/null +++ b/doc/man/restic-repair-packs.1 @@ -0,0 +1,122 @@ +.nh +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" + +.SH NAME +.PP +restic-repair-packs - Salvage damaged pack files + + +.SH SYNOPSIS +.PP +\fBrestic repair packs [packIDs...] [flags]\fP + + +.SH DESCRIPTION +.PP +WARNING: The CLI for this command is experimental and will likely change in the future! + +.PP +The "repair packs" command extracts intact blobs from the specified pack files, rebuilds +the index to remove the damaged pack files and removes the pack files from the repository. + + +.SH EXIT STATUS +.PP +Exit status is 0 if the command was successful, and non-zero if there was any error. + + +.SH OPTIONS +.PP +\fB-h\fP, \fB--help\fP[=false] + help for packs + + +.SH OPTIONS INHERITED FROM PARENT COMMANDS +.PP +\fB--cacert\fP=[] + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) + +.PP +\fB--cache-dir\fP="" + set the cache \fBdirectory\fR\&. (default: use system default cache directory) + +.PP +\fB--cleanup-cache\fP[=false] + auto remove old cache directories + +.PP +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) + +.PP +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] + set output mode to JSON for commands that support it + +.PP +\fB--key-hint\fP="" + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + +.PP +\fB--limit-download\fP=0 + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--limit-upload\fP=0 + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--no-cache\fP[=false] + do not use a local cache + +.PP +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories + +.PP +\fB-o\fP, \fB--option\fP=[] + set extended option (\fBkey=value\fR, can be specified multiple times) + +.PP +\fB--pack-size\fP=0 + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + +.PP +\fB-p\fP, \fB--password-file\fP="" + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + +.PP +\fB-q\fP, \fB--quiet\fP[=false] + do not output comprehensive progress report + +.PP +\fB-r\fP, \fB--repo\fP="" + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + +.PP +\fB--repository-file\fP="" + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) + +.PP +\fB--tls-client-cert\fP="" + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) + +.PP +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) + + +.SH SEE ALSO +.PP +\fBrestic-repair(1)\fP diff --git a/doc/man/restic-repair-snapshots.1 b/doc/man/restic-repair-snapshots.1 new file mode 100644 index 000000000..f59067f05 --- /dev/null +++ b/doc/man/restic-repair-snapshots.1 @@ -0,0 +1,157 @@ +.nh +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" + +.SH NAME +.PP +restic-repair-snapshots - Repair snapshots + + +.SH SYNOPSIS +.PP +\fBrestic repair snapshots [flags] [snapshot ID] [...]\fP + + +.SH DESCRIPTION +.PP +The "repair snapshots" command repairs broken snapshots. It scans the given +snapshots and generates new ones with damaged directories and file contents +removed. If the broken snapshots are deleted, a prune run will be able to +clean up the repository. + +.PP +The command depends on a correct index, thus make sure to run "repair index" +first! + + +.SH WARNING +.PP +Repairing and deleting broken snapshots causes data loss! It will remove broken +directories and modify broken files in the modified snapshots. + +.PP +If the contents of directories and files are still available, the better option +is to run "backup" which in that case is able to heal existing snapshots. Only +use the "repair snapshots" command if you need to recover an old and broken +snapshot! + + +.SH EXIT STATUS +.PP +Exit status is 0 if the command was successful, and non-zero if there was any error. + + +.SH OPTIONS +.PP +\fB-n\fP, \fB--dry-run\fP[=false] + do not do anything, just print what would be done + +.PP +\fB--forget\fP[=false] + remove original snapshots after creating new ones + +.PP +\fB-h\fP, \fB--help\fP[=false] + help for snapshots + +.PP +\fB-H\fP, \fB--host\fP=[] + only consider snapshots for this \fBhost\fR (can be specified multiple times) + +.PP +\fB--path\fP=[] + only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times) + +.PP +\fB--tag\fP=[] + only consider snapshots including \fBtag[,tag,...]\fR (can be specified multiple times) + + +.SH OPTIONS INHERITED FROM PARENT COMMANDS +.PP +\fB--cacert\fP=[] + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) + +.PP +\fB--cache-dir\fP="" + set the cache \fBdirectory\fR\&. (default: use system default cache directory) + +.PP +\fB--cleanup-cache\fP[=false] + auto remove old cache directories + +.PP +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) + +.PP +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] + set output mode to JSON for commands that support it + +.PP +\fB--key-hint\fP="" + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + +.PP +\fB--limit-download\fP=0 + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--limit-upload\fP=0 + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--no-cache\fP[=false] + do not use a local cache + +.PP +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories + +.PP +\fB-o\fP, \fB--option\fP=[] + set extended option (\fBkey=value\fR, can be specified multiple times) + +.PP +\fB--pack-size\fP=0 + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + +.PP +\fB-p\fP, \fB--password-file\fP="" + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + +.PP +\fB-q\fP, \fB--quiet\fP[=false] + do not output comprehensive progress report + +.PP +\fB-r\fP, \fB--repo\fP="" + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + +.PP +\fB--repository-file\fP="" + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) + +.PP +\fB--tls-client-cert\fP="" + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) + +.PP +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) + + +.SH SEE ALSO +.PP +\fBrestic-repair(1)\fP diff --git a/doc/man/restic-repair.1 b/doc/man/restic-repair.1 new file mode 100644 index 000000000..dbe783df4 --- /dev/null +++ b/doc/man/restic-repair.1 @@ -0,0 +1,113 @@ +.nh +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" + +.SH NAME +.PP +restic-repair - Repair the repository + + +.SH SYNOPSIS +.PP +\fBrestic repair [flags]\fP + + +.SH DESCRIPTION +.PP +Repair the repository + + +.SH OPTIONS +.PP +\fB-h\fP, \fB--help\fP[=false] + help for repair + + +.SH OPTIONS INHERITED FROM PARENT COMMANDS +.PP +\fB--cacert\fP=[] + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) + +.PP +\fB--cache-dir\fP="" + set the cache \fBdirectory\fR\&. (default: use system default cache directory) + +.PP +\fB--cleanup-cache\fP[=false] + auto remove old cache directories + +.PP +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) + +.PP +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] + set output mode to JSON for commands that support it + +.PP +\fB--key-hint\fP="" + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + +.PP +\fB--limit-download\fP=0 + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--limit-upload\fP=0 + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--no-cache\fP[=false] + do not use a local cache + +.PP +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories + +.PP +\fB-o\fP, \fB--option\fP=[] + set extended option (\fBkey=value\fR, can be specified multiple times) + +.PP +\fB--pack-size\fP=0 + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + +.PP +\fB-p\fP, \fB--password-file\fP="" + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + +.PP +\fB-q\fP, \fB--quiet\fP[=false] + do not output comprehensive progress report + +.PP +\fB-r\fP, \fB--repo\fP="" + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + +.PP +\fB--repository-file\fP="" + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) + +.PP +\fB--tls-client-cert\fP="" + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) + +.PP +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) + + +.SH SEE ALSO +.PP +\fBrestic(1)\fP, \fBrestic-repair-index(1)\fP, \fBrestic-repair-packs(1)\fP, \fBrestic-repair-snapshots(1)\fP diff --git a/doc/man/restic-restore.1 b/doc/man/restic-restore.1 index 39ff62059..d8c1b72e1 100644 --- a/doc/man/restic-restore.1 +++ b/doc/man/restic-restore.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -17,9 +17,13 @@ The "restore" command extracts the data from a snapshot from the repository to a directory. .PP -The special snapshot "latest" can be used to restore the latest snapshot in the +The special snapshotID "latest" can be used to restore the latest snapshot in the repository. +.PP +To only restore a specific subfolder, you can use the ":" +syntax, where "subfolder" is a path within the snapshot. + .SH EXIT STATUS .PP @@ -29,7 +33,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .SH OPTIONS .PP \fB-e\fP, \fB--exclude\fP=[] - exclude a \fB\fCpattern\fR (can be specified multiple times) + exclude a \fBpattern\fR (can be specified multiple times) .PP \fB-h\fP, \fB--help\fP[=false] @@ -37,23 +41,23 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fB\fChost\fR, when snapshot ID "latest" is given (can be specified multiple times) + only consider snapshots for this \fBhost\fR, when snapshot ID "latest" is given (can be specified multiple times) .PP \fB--iexclude\fP=[] - same as \fB\fC--exclude\fR but ignores the casing of filenames + same as --exclude but ignores the casing of \fBpattern\fR .PP \fB--iinclude\fP=[] - same as \fB\fC--include\fR but ignores the casing of filenames + same as --include but ignores the casing of \fBpattern\fR .PP \fB-i\fP, \fB--include\fP=[] - include a \fB\fCpattern\fR, exclude everything else (can be specified multiple times) + include a \fBpattern\fR, exclude everything else (can be specified multiple times) .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fB\fCpath\fR, when snapshot ID "latest" is given (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR, when snapshot ID "latest" is given (can be specified multiple times) .PP \fB--sparse\fP[=false] @@ -61,7 +65,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--tag\fP=[] - only consider snapshots including \fB\fCtag[,tag,...]\fR, when snapshot ID "latest" is given (can be specified multiple times) + only consider snapshots including \fBtag[,tag,...]\fR, when snapshot ID "latest" is given (can be specified multiple times) .PP \fB-t\fP, \fB--target\fP="" @@ -75,11 +79,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -87,7 +91,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -99,15 +103,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -119,19 +123,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -139,15 +143,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-rewrite.1 b/doc/man/restic-rewrite.1 index 6edf51b95..8a06aef40 100644 --- a/doc/man/restic-rewrite.1 +++ b/doc/man/restic-rewrite.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -45,11 +45,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-e\fP, \fB--exclude\fP=[] - exclude a \fB\fCpattern\fR (can be specified multiple times) + exclude a \fBpattern\fR (can be specified multiple times) .PP \fB--exclude-file\fP=[] - read exclude patterns from a \fB\fCfile\fR (can be specified multiple times) + read exclude patterns from a \fBfile\fR (can be specified multiple times) .PP \fB--forget\fP[=false] @@ -61,33 +61,33 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fB\fChost\fR (can be specified multiple times) + only consider snapshots for this \fBhost\fR (can be specified multiple times) .PP \fB--iexclude\fP=[] - same as --exclude \fB\fCpattern\fR but ignores the casing of filenames + same as --exclude \fBpattern\fR but ignores the casing of filenames .PP \fB--iexclude-file\fP=[] - same as --exclude-file but ignores casing of \fB\fCfile\fRnames in patterns + same as --exclude-file but ignores casing of \fBfile\fRnames in patterns .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fB\fCpath\fR (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times) .PP \fB--tag\fP=[] - only consider snapshots including \fB\fCtag[,tag,...]\fR (can be specified multiple times) + only consider snapshots including \fBtag[,tag,...]\fR (can be specified multiple times) .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -95,7 +95,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -107,15 +107,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -127,19 +127,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -147,15 +147,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-self-update.1 b/doc/man/restic-self-update.1 index e311b2277..28fd24a92 100644 --- a/doc/man/restic-self-update.1 +++ b/doc/man/restic-self-update.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -31,17 +31,17 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--output\fP="" - Save the downloaded file as \fB\fCfilename\fR (default: running binary itself) + Save the downloaded file as \fBfilename\fR (default: running binary itself) .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -49,7 +49,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -61,15 +61,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -81,19 +81,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -101,15 +101,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-snapshots.1 b/doc/man/restic-snapshots.1 index d2dbf52ee..cb34d6c8e 100644 --- a/doc/man/restic-snapshots.1 +++ b/doc/man/restic-snapshots.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -27,8 +27,8 @@ Exit status is 0 if the command was successful, and non-zero if there was any er use compact output format .PP -\fB-g\fP, \fB--group-by\fP="" - \fB\fCgroup\fR snapshots by host, paths and/or tags, separated by comma +\fB-g\fP, \fB--group-by\fP= + \fBgroup\fR snapshots by host, paths and/or tags, separated by comma .PP \fB-h\fP, \fB--help\fP[=false] @@ -36,29 +36,29 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fB\fChost\fR (can be specified multiple times) + only consider snapshots for this \fBhost\fR (can be specified multiple times) .PP \fB--latest\fP=0 - only show the last \fB\fCn\fR snapshots for each host and path + only show the last \fBn\fR snapshots for each host and path .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fB\fCpath\fR (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times) .PP \fB--tag\fP=[] - only consider snapshots including \fB\fCtag[,tag,...]\fR (can be specified multiple times) + only consider snapshots including \fBtag[,tag,...]\fR (can be specified multiple times) .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -66,7 +66,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -78,15 +78,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -98,19 +98,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -118,15 +118,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-stats.1 b/doc/man/restic-stats.1 index 694bde22d..cf0374351 100644 --- a/doc/man/restic-stats.1 +++ b/doc/man/restic-stats.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -58,7 +58,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fB\fChost\fR (can be specified multiple times) + only consider snapshots for this \fBhost\fR (can be specified multiple times) .PP \fB--mode\fP="restore-size" @@ -66,21 +66,21 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fB\fCpath\fR (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times) .PP \fB--tag\fP=[] - only consider snapshots including \fB\fCtag[,tag,...]\fR (can be specified multiple times) + only consider snapshots including \fBtag[,tag,...]\fR (can be specified multiple times) .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -88,7 +88,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -100,15 +100,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -120,19 +120,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -140,15 +140,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-tag.1 b/doc/man/restic-tag.1 index 1ff0b4f78..162d50d29 100644 --- a/doc/man/restic-tag.1 +++ b/doc/man/restic-tag.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -8,7 +8,7 @@ restic-tag - Modify tags on snapshots .SH SYNOPSIS .PP -\fBrestic tag [flags] [snapshot-ID ...]\fP +\fBrestic tag [flags] [snapshotID ...]\fP .SH DESCRIPTION @@ -20,7 +20,7 @@ You can either set/replace the entire set of tags on a snapshot, or add tags to/remove tags from the existing set. .PP -When no snapshot-ID is given, all snapshots matching the host, tag and path filter criteria are modified. +When no snapshotID is given, all snapshots matching the host, tag and path filter criteria are modified. .SH EXIT STATUS @@ -31,7 +31,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .SH OPTIONS .PP \fB--add\fP=[] - \fB\fCtags\fR which will be added to the existing tags in the format \fB\fCtag[,tag,...]\fR (can be given multiple times) + \fBtags\fR which will be added to the existing tags in the format \fBtag[,tag,...]\fR (can be given multiple times) .PP \fB-h\fP, \fB--help\fP[=false] @@ -39,33 +39,33 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fB\fChost\fR (can be specified multiple times) + only consider snapshots for this \fBhost\fR (can be specified multiple times) .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fB\fCpath\fR (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times) .PP \fB--remove\fP=[] - \fB\fCtags\fR which will be removed from the existing tags in the format \fB\fCtag[,tag,...]\fR (can be given multiple times) + \fBtags\fR which will be removed from the existing tags in the format \fBtag[,tag,...]\fR (can be given multiple times) .PP \fB--set\fP=[] - \fB\fCtags\fR which will replace the existing tags in the format \fB\fCtag[,tag,...]\fR (can be given multiple times) + \fBtags\fR which will replace the existing tags in the format \fBtag[,tag,...]\fR (can be given multiple times) .PP \fB--tag\fP=[] - only consider snapshots including \fB\fCtag[,tag,...]\fR (can be specified multiple times) + only consider snapshots including \fBtag[,tag,...]\fR (can be specified multiple times) .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -73,7 +73,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -85,15 +85,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -105,19 +105,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -125,15 +125,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-unlock.1 b/doc/man/restic-unlock.1 index e5b408915..0274c56e8 100644 --- a/doc/man/restic-unlock.1 +++ b/doc/man/restic-unlock.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -34,11 +34,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -46,7 +46,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -58,15 +58,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -78,19 +78,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -98,15 +98,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic-version.1 b/doc/man/restic-version.1 index eca34d60a..774e19453 100644 --- a/doc/man/restic-version.1 +++ b/doc/man/restic-version.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -31,11 +31,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -43,7 +43,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB--insecure-tls\fP[=false] @@ -55,15 +55,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -75,19 +75,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -95,15 +95,19 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] diff --git a/doc/man/restic.1 b/doc/man/restic.1 index f76d16e38..427ce7c65 100644 --- a/doc/man/restic.1 +++ b/doc/man/restic.1 @@ -1,5 +1,5 @@ .nh -.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" "" +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" .SH NAME .PP @@ -16,15 +16,18 @@ restic - Backup and restore files restic is a backup program which allows saving multiple revisions of files and directories in an encrypted repository stored on different backends. +.PP +The full documentation can be found at https://restic.readthedocs.io/ . + .SH OPTIONS .PP \fB--cacert\fP=[] - \fB\fCfile\fR to load root certificates from (default: use system certificates) + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) .PP \fB--cache-dir\fP="" - set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) + set the cache \fBdirectory\fR\&. (default: use system default cache directory) .PP \fB--cleanup-cache\fP[=false] @@ -32,7 +35,7 @@ directories in an encrypted repository stored on different backends. .PP \fB--compression\fP=auto - compression mode (only available for repository format version 2), one of (auto|off|max) + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) .PP \fB-h\fP, \fB--help\fP[=false] @@ -48,15 +51,15 @@ directories in an encrypted repository stored on different backends. .PP \fB--key-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP \fB--limit-download\fP=0 - limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--limit-upload\fP=0 - limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited) + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) .PP \fB--no-cache\fP[=false] @@ -68,19 +71,19 @@ directories in an encrypted repository stored on different backends. .PP \fB-o\fP, \fB--option\fP=[] - set extended option (\fB\fCkey=value\fR, can be specified multiple times) + set extended option (\fBkey=value\fR, can be specified multiple times) .PP \fB--pack-size\fP=0 - set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) .PP \fB--password-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP \fB-p\fP, \fB--password-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP \fB-q\fP, \fB--quiet\fP[=false] @@ -88,15 +91,19 @@ directories in an encrypted repository stored on different backends. .PP \fB-r\fP, \fB--repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP \fB--repository-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) .PP \fB--tls-client-cert\fP="" - path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) .PP \fB-v\fP, \fB--verbose\fP[=0] @@ -105,4 +112,4 @@ directories in an encrypted repository stored on different backends. .SH SEE ALSO .PP -\fBrestic-backup(1)\fP, \fBrestic-cache(1)\fP, \fBrestic-cat(1)\fP, \fBrestic-check(1)\fP, \fBrestic-copy(1)\fP, \fBrestic-diff(1)\fP, \fBrestic-dump(1)\fP, \fBrestic-find(1)\fP, \fBrestic-forget(1)\fP, \fBrestic-generate(1)\fP, \fBrestic-init(1)\fP, \fBrestic-key(1)\fP, \fBrestic-list(1)\fP, \fBrestic-ls(1)\fP, \fBrestic-migrate(1)\fP, \fBrestic-mount(1)\fP, \fBrestic-prune(1)\fP, \fBrestic-rebuild-index(1)\fP, \fBrestic-recover(1)\fP, \fBrestic-restore(1)\fP, \fBrestic-rewrite(1)\fP, \fBrestic-self-update(1)\fP, \fBrestic-snapshots(1)\fP, \fBrestic-stats(1)\fP, \fBrestic-tag(1)\fP, \fBrestic-unlock(1)\fP, \fBrestic-version(1)\fP +\fBrestic-backup(1)\fP, \fBrestic-cache(1)\fP, \fBrestic-cat(1)\fP, \fBrestic-check(1)\fP, \fBrestic-copy(1)\fP, \fBrestic-diff(1)\fP, \fBrestic-dump(1)\fP, \fBrestic-find(1)\fP, \fBrestic-forget(1)\fP, \fBrestic-generate(1)\fP, \fBrestic-init(1)\fP, \fBrestic-key(1)\fP, \fBrestic-list(1)\fP, \fBrestic-ls(1)\fP, \fBrestic-migrate(1)\fP, \fBrestic-mount(1)\fP, \fBrestic-prune(1)\fP, \fBrestic-recover(1)\fP, \fBrestic-repair(1)\fP, \fBrestic-restore(1)\fP, \fBrestic-rewrite(1)\fP, \fBrestic-self-update(1)\fP, \fBrestic-snapshots(1)\fP, \fBrestic-stats(1)\fP, \fBrestic-tag(1)\fP, \fBrestic-unlock(1)\fP, \fBrestic-version(1)\fP diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index 71f5e192b..d1c64ba6e 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -148,11 +148,11 @@ command: -v, --verbose be verbose (specify multiple times or a level using --verbose=n, max level/times is 2) Subcommands that support showing progress information such as ``backup``, -``check`` and ``prune`` will do so unless the quiet flag ``-q`` or -``--quiet`` is set. When running from a non-interactive console progress -reporting is disabled by default to not fill your logs. For interactive -and non-interactive consoles the environment variable ``RESTIC_PROGRESS_FPS`` -can be used to control the frequency of progress reporting. Use for example +``restore``, ``check`` and ``prune`` will do so unless the quiet flag ``-q`` +or ``--quiet`` is set. When running from a non-interactive console progress +reporting is disabled by default to not fill your logs. For interactive and +non-interactive consoles the environment variable ``RESTIC_PROGRESS_FPS`` can +be used to control the frequency of progress reporting. Use for example ``0.016666`` to only update the progress once per minute. Additionally, on Unix systems if ``restic`` receives a SIGUSR1 signal the diff --git a/doc/powershell-completion.ps1 b/doc/powershell-completion.ps1 index 271809161..d8aa5a1af 100644 --- a/doc/powershell-completion.ps1 +++ b/doc/powershell-completion.ps1 @@ -40,6 +40,7 @@ filter __restic_escapeStringWithSpecialChars { $ShellCompDirectiveNoFileComp=4 $ShellCompDirectiveFilterFileExt=8 $ShellCompDirectiveFilterDirs=16 + $ShellCompDirectiveKeepOrder=32 # Prepare the command to request completions for the program. # Split the command at the first space to separate the program and arguments. @@ -69,8 +70,17 @@ filter __restic_escapeStringWithSpecialChars { # If the last parameter is complete (there is a space following it) # We add an extra empty parameter so we can indicate this to the go method. __restic_debug "Adding extra empty parameter" - # We need to use `"`" to pass an empty argument a "" or '' does not work!!! - $RequestComp="$RequestComp" + ' `"`"' + # PowerShell 7.2+ changed the way how the arguments are passed to executables, + # so for pre-7.2 or when Legacy argument passing is enabled we need to use + # `"`" to pass an empty argument, a "" or '' does not work!!! + if ($PSVersionTable.PsVersion -lt [version]'7.2.0' -or + ($PSVersionTable.PsVersion -lt [version]'7.3.0' -and -not [ExperimentalFeature]::IsEnabled("PSNativeCommandArgumentPassing")) -or + (($PSVersionTable.PsVersion -ge [version]'7.3.0' -or [ExperimentalFeature]::IsEnabled("PSNativeCommandArgumentPassing")) -and + $PSNativeCommandArgumentPassing -eq 'Legacy')) { + $RequestComp="$RequestComp" + ' `"`"' + } else { + $RequestComp="$RequestComp" + ' ""' + } } __restic_debug "Calling $RequestComp" @@ -100,7 +110,7 @@ filter __restic_escapeStringWithSpecialChars { } $Longest = 0 - $Values = $Out | ForEach-Object { + [Array]$Values = $Out | ForEach-Object { #Split the output in name and description $Name, $Description = $_.Split("`t",2) __restic_debug "Name: $Name Description: $Description" @@ -145,6 +155,11 @@ filter __restic_escapeStringWithSpecialChars { } } + # we sort the values in ascending order by name if keep order isn't passed + if (($Directive -band $ShellCompDirectiveKeepOrder) -eq 0 ) { + $Values = $Values | Sort-Object -Property Name + } + if (($Directive -band $ShellCompDirectiveNoFileComp) -ne 0 ) { __restic_debug "ShellCompDirectiveNoFileComp is called" diff --git a/doc/zsh-completion.zsh b/doc/zsh-completion.zsh index cea6abb38..ec1c52a00 100644 --- a/doc/zsh-completion.zsh +++ b/doc/zsh-completion.zsh @@ -1,4 +1,5 @@ #compdef restic +compdef _restic restic # zsh completion for restic -*- shell-script -*- @@ -17,8 +18,9 @@ _restic() local shellCompDirectiveNoFileComp=4 local shellCompDirectiveFilterFileExt=8 local shellCompDirectiveFilterDirs=16 + local shellCompDirectiveKeepOrder=32 - local lastParam lastChar flagPrefix requestComp out directive comp lastComp noSpace + local lastParam lastChar flagPrefix requestComp out directive comp lastComp noSpace keepOrder local -a completions __restic_debug "\n========= starting completion logic ==========" @@ -136,6 +138,11 @@ _restic() noSpace="-S ''" fi + if [ $((directive & shellCompDirectiveKeepOrder)) -ne 0 ]; then + __restic_debug "Activating keep order." + keepOrder="-V" + fi + if [ $((directive & shellCompDirectiveFilterFileExt)) -ne 0 ]; then # File extension filtering local filteringCmd @@ -171,7 +178,7 @@ _restic() return $result else __restic_debug "Calling _describe" - if eval _describe "completions" completions $flagPrefix $noSpace; then + if eval _describe $keepOrder "completions" completions $flagPrefix $noSpace; then __restic_debug "_describe found some completions" # Return the success of having called _describe diff --git a/docker/Dockerfile b/docker/Dockerfile index ecc283f8a..978da7960 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,7 +11,7 @@ RUN go run build.go FROM alpine:latest AS restic -RUN apk add --update --no-cache ca-certificates fuse openssh-client tzdata jq +RUN apk add --no-cache ca-certificates fuse openssh-client tzdata jq COPY --from=builder /go/src/github.com/restic/restic/restic /usr/bin diff --git a/docker/Dockerfile.release b/docker/Dockerfile.release index ccf80376a..42302f203 100644 --- a/docker/Dockerfile.release +++ b/docker/Dockerfile.release @@ -13,6 +13,6 @@ RUN mv /output/restic_${TARGETOS}_${TARGETARCH} /output/restic FROM alpine:latest COPY --from=helper /output/restic /usr/bin -RUN apk add --update --no-cache ca-certificates fuse openssh-client tzdata jq +RUN apk add --no-cache ca-certificates fuse openssh-client tzdata jq ENTRYPOINT ["/usr/bin/restic"] diff --git a/go.mod b/go.mod index 6968a2f7f..4756bdf01 100644 --- a/go.mod +++ b/go.mod @@ -1,64 +1,66 @@ module github.com/restic/restic require ( - cloud.google.com/go/storage v1.30.1 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 + cloud.google.com/go/storage v1.34.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 + github.com/Backblaze/blazer v0.6.1 github.com/anacrolix/fuse v0.2.0 - github.com/cenkalti/backoff/v4 v4.2.0 + github.com/cenkalti/backoff/v4 v4.2.1 github.com/cespare/xxhash/v2 v2.2.0 github.com/elithrar/simple-scrypt v1.3.0 - github.com/go-ole/go-ole v1.2.6 - github.com/google/go-cmp v0.5.9 - github.com/google/uuid v1.3.0 - github.com/hashicorp/golang-lru/v2 v2.0.1 + github.com/go-ole/go-ole v1.3.0 + github.com/google/go-cmp v0.6.0 + github.com/google/uuid v1.5.0 + github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/hirochachacha/go-smb2 v1.1.0 - github.com/juju/ratelimit v1.0.2 - github.com/klauspost/compress v1.16.5 - github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5 - github.com/minio/minio-go/v7 v7.0.56 + github.com/klauspost/compress v1.17.2 + github.com/minio/minio-go/v7 v7.0.63 github.com/minio/sha256-simd v1.0.1 - github.com/ncw/swift/v2 v2.0.1 + github.com/ncw/swift/v2 v2.0.2 github.com/pkg/errors v0.9.1 github.com/pkg/profile v1.7.0 - github.com/pkg/sftp v1.13.5 + github.com/pkg/sftp v1.13.6 github.com/pkg/xattr v0.4.10-0.20221120235825-35026bbbd013 github.com/restic/chunker v0.4.0 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 - golang.org/x/crypto v0.10.0 - golang.org/x/net v0.11.0 - golang.org/x/oauth2 v0.9.0 - golang.org/x/sync v0.3.0 - golang.org/x/sys v0.9.0 - golang.org/x/term v0.9.0 - golang.org/x/text v0.10.0 - google.golang.org/api v0.129.0 + go.uber.org/automaxprocs v1.5.3 + golang.org/x/crypto v0.16.0 + golang.org/x/net v0.19.0 + golang.org/x/oauth2 v0.15.0 + golang.org/x/sync v0.5.0 + golang.org/x/sys v0.15.0 + golang.org/x/term v0.15.0 + golang.org/x/text v0.14.0 + golang.org/x/time v0.5.0 + google.golang.org/api v0.149.0 ) require ( - cloud.google.com/go v0.110.0 // indirect - cloud.google.com/go/compute v1.19.3 // indirect + cloud.google.com/go v0.110.9 // indirect + cloud.google.com/go/compute v1.23.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.13.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + cloud.google.com/go/iam v1.1.3 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.4.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/fgprof v0.9.3 // indirect github.com/geoffgarside/ber v1.1.0 // indirect - github.com/golang-jwt/jwt/v4 v4.5.0 // indirect + github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b // indirect - github.com/google/s2a-go v0.1.4 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect - github.com/googleapis/gax-go/v2 v2.11.0 // indirect + github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kr/fs v0.1.0 // indirect + github.com/kr/text v0.2.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -66,17 +68,17 @@ require ( github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/rs/xid v1.5.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sirupsen/logrus v1.9.2 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect - google.golang.org/grpc v1.56.1 // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect + google.golang.org/grpc v1.59.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) -go 1.18 +go 1.19 diff --git a/go.sum b/go.sum index 6ba1696df..0766c1237 100644 --- a/go.sum +++ b/go.sum @@ -1,34 +1,34 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= -cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= -cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds= -cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= +cloud.google.com/go v0.110.9 h1:e7ITSqGFFk4rbz/JFIqZh3G4VEHguhAL4BQcFlWtU68= +cloud.google.com/go v0.110.9/go.mod h1:rpxevX/0Lqvlbc88b7Sc1SPNdyK1riNBTUU6JXhYNpM= +cloud.google.com/go/compute v1.23.1 h1:V97tBoDaZHb6leicZ1G6DLK2BAaZLJ/7+9BB/En3hR0= +cloud.google.com/go/compute v1.23.1/go.mod h1:CqB3xpmPKKt3OJpW2ndFIXnA9A4xAy/F3Xp1ixncW78= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= -cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= -cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= -cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 h1:SEy2xmstIphdPwNBUi7uhvjyjhVKISfwjfOJmuy7kg4= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= -github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= -github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= +cloud.google.com/go/iam v1.1.3 h1:18tKG7DzydKWUnLjonWcJO6wjSCAtzh4GcRKlH/Hrzc= +cloud.google.com/go/iam v1.1.3/go.mod h1:3khUlaBXfPKKe7huYgEpDn6FtgRyMEqbkvBxrQyY5SE= +cloud.google.com/go/storage v1.34.0 h1:9KHBBTbaHPsNxO043SFmH3pMojjZiW+BFl9H41L7xjk= +cloud.google.com/go/storage v1.34.0/go.mod h1:Eji+S0CCQebjsiXxyIvPItC3BN3zWsdJjWfHfoLblgY= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 h1:9kDVnTz3vbfweTqAUmk/a/pH5pWFCHtvRpHYC0G/dcA= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0/go.mod h1:3Ug6Qzto9anB6mGlEdgYMDF5zHQ+wwhEaYR4s17PHMw= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.4.0 h1:TuEMD+E+1aTjjLICGQOW6vLe8UWES7kopac9mUXL56Y= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.4.0/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 h1:gggzg0SUMs6SQbEw+3LoSsYf9YMjkupeAnHMX8O9mmY= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0/go.mod h1:+6KLcKIVgxoBDMqMO/Nvy7bZ9a0nbU3I1DtFQK3YvB4= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 h1:hVeq+yCyUi+MsoO/CU95yqCIcdzra5ovzk8Q2BBpV2M= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/Backblaze/blazer v0.6.1 h1:xC9HyC7OcxRzzmtfRiikIEvq4HZYWjU6caFwX2EXw1s= +github.com/Backblaze/blazer v0.6.1/go.mod h1:7/jrGx4O6OKOto6av+hLwelPR8rwZ+PLxQ5ZOiYAjwY= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/anacrolix/fuse v0.2.0 h1:pc+To78kI2d/WUjIyrsdqeJQAesuwpGxlI3h1nAv3Do= github.com/anacrolix/fuse v0.2.0/go.mod h1:Kfu02xBwnySDpH3N23BmrP3MDfwAQGRLUCj6XyeOvBQ= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= -github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -36,13 +36,10 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -56,34 +53,28 @@ github.com/elithrar/simple-scrypt v1.3.0/go.mod h1:U2XQRI95XHY0St410VE3UjT7vuKb1 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= -github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= +github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= @@ -96,25 +87,24 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= -github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b h1:8htHrh2bw9c7Idkb7YNac+ZpTqLMjRpI+FWu51ltaQc= -github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= -github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= -github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ= +github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= -github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= -github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= -github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= -github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI= github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= @@ -122,23 +112,22 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI= -github.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5 h1:OUlGa6AAolmjyPtILbMJ8vHayz5wd4wBUloheGcMhfA= -github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.56 h1:pkZplIEHu8vinjkmhsexcXpWth2tjVLphrTZx6fBVZY= -github.com/minio/minio-go/v7 v7.0.56/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE= +github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ= +github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -146,31 +135,31 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/ncw/swift/v2 v2.0.1 h1:q1IN8hNViXEv8Zvg3Xdis4a3c4IlIGezkYz09zQL5J0= -github.com/ncw/swift/v2 v2.0.1/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg= +github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk= +github.com/ncw/swift/v2 v2.0.2/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo= -github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go= -github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg= +github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo= +github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= github.com/pkg/xattr v0.4.10-0.20221120235825-35026bbbd013 h1:aqByeeNnF7NiEbXCi7nBxZ272+6f6FUBmj/dUzWCdvc= github.com/pkg/xattr v0.4.10-0.20221120235825-35026bbbd013/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/restic/chunker v0.4.0 h1:YUPYCUn70MYP7VO4yllypp2SjmsRhRJaad3xKu1QFRw= github.com/restic/chunker v0.4.0/go.mod h1:z0cH2BejpW636LXw0R/BGyv+Ey8+m9QGiOanDHItzyw= github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= -github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -180,28 +169,27 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= +go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -210,64 +198,58 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs= -golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw= +golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= +golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= -golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -279,35 +261,30 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/api v0.129.0 h1:2XbdjjNfFPXQyufzQVwPf1RRnHH8Den2pfNE2jw7L8w= -google.golang.org/api v0.129.0/go.mod h1:dFjiXlanKwWE3612X97llhsoI36FAoIiRj3aTl5b/zE= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +google.golang.org/api v0.149.0 h1:b2CqT6kG+zqJIVKRQ3ELJVLN1PwHZ6DJ3dW8yl82rgY= +google.golang.org/api v0.149.0/go.mod h1:Mwn1B7JTXrzXtnvmzQE2BD6bYZQ8DShKZDZbeN9I7qI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao= -google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= -google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM= -google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA= +google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI= +google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k= +google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= -google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -321,12 +298,10 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/helpers/build-release-binaries/main.go b/helpers/build-release-binaries/main.go index caa90ff82..81d126b00 100644 --- a/helpers/build-release-binaries/main.go +++ b/helpers/build-release-binaries/main.go @@ -125,6 +125,10 @@ func build(sourceDir, outputDir, goos, goarch string) (filename string) { "GOOS="+goos, "GOARCH="+goarch, ) + if goarch == "arm" { + // the raspberry pi 1 only supports the ARMv6 instruction set + c.Env = append(c.Env, "GOARM=6") + } verbose("run %v %v in %v", "go", c.Args, c.Dir) err := c.Run() @@ -239,6 +243,7 @@ func buildTargets(sourceDir, outputDir string, targets map[string][]string) { } var defaultBuildTargets = map[string][]string{ + "aix": {"ppc64"}, "darwin": {"amd64", "arm64"}, "freebsd": {"386", "amd64", "arm"}, "linux": {"386", "amd64", "arm", "arm64", "ppc64le", "mips", "mipsle", "mips64", "mips64le", "riscv64", "s390x"}, diff --git a/helpers/prepare-release/main.go b/helpers/prepare-release/main.go index a6c7bd4f4..baf8aa2ba 100644 --- a/helpers/prepare-release/main.go +++ b/helpers/prepare-release/main.go @@ -379,7 +379,7 @@ func readdir(dir string) []string { } func sha256sums(inputDir, outputFile string) { - msg("runnnig sha256sum in %v", inputDir) + msg("running sha256sum in %v", inputDir) filenames := readdir(inputDir) diff --git a/helpers/verify-release-binaries.sh b/helpers/verify-release-binaries.sh new file mode 100755 index 000000000..4e80528e2 --- /dev/null +++ b/helpers/verify-release-binaries.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +set -euo pipefail + +if [[ $# -lt 2 ]]; then + echo "Usage: $0 restic_version go_version" + exit 1 +fi + +restic_version="$1" +go_version="$2" + +# invalid if zero +is_valid=1 +set_invalid() { + echo $1 + is_valid=0 +} + +tmpdir="$(mktemp -d -p .)" +cd "${tmpdir}" +echo -e "Running checks in ${tmpdir}\n" + +highlight() { + echo "@@${1//?/@}@@" + echo "@ ${1} @" + echo "@@${1//?/@}@@" +} + + +highlight "Verifying release self-consistency" + +curl -OLSs https://github.com/restic/restic/releases/download/v${restic_version}/restic-${restic_version}.tar.gz.asc +# tarball is downloaded while processing the SHA256SUMS +curl -OLSs https://github.com/restic/restic/releases/download/v${restic_version}/SHA256SUMS.asc +curl -OLSs https://github.com/restic/restic/releases/download/v${restic_version}/SHA256SUMS + +export GNUPGHOME=$PWD/gnupg +mkdir -p 700 $GNUPGHOME +curl -OLSs https://restic.net/gpg-key-alex.asc +gpg --import gpg-key-alex.asc +gpg --verify SHA256SUMS.asc SHA256SUMS + +for i in $(cat SHA256SUMS | cut -d " " -f 3 ) ; do + echo "Downloading $i" + curl -OLSs https://github.com/restic/restic/releases/download/v${restic_version}/"$i" +done +shasum -a256 -c SHA256SUMS || set_invalid "WARNING: RELEASE BINARIES DO NOT MATCH SHA256SUMS!" +gpg --verify restic-${restic_version}.tar.gz.asc restic-${restic_version}.tar.gz +# TODO verify that the release does not contain any unexpected files + + +highlight "Verifying tarball matches tagged commit" + +tar xzf "restic-${restic_version}.tar.gz" +git clone -b "v${restic_version}" https://github.com/restic/restic.git +rm -rf restic/.git +diff -r restic restic-${restic_version} + + +highlight "Regenerating builder container" + +git clone https://github.com/restic/builder.git +docker pull debian:stable +docker build --no-cache -t restic/builder:tmp --build-arg GO_VERSION=${go_version} builder + + +highlight "Reproducing release binaries" + +mkdir output +docker run --rm \ + --volume "$PWD/restic-${restic_version}:/restic" \ + --volume "$PWD/output:/output" \ + restic/builder:tmp \ + go run helpers/build-release-binaries/main.go --version "${restic_version}" + +cp "restic-${restic_version}.tar.gz" output +cp SHA256SUMS output + +# check that all release binaries have been reproduced successfully +(cd output && shasum -a256 -c SHA256SUMS) || set_invalid "WARNING: REPRODUCED BINARIES DO NOT MATCH RELEASE BINARIES!" +# and that the SHA256SUMS files does not miss binaries +for i in output/restic* ; do grep "$(basename "$i")" SHA256SUMS > /dev/null || set_invalid "WARNING: $i MISSING FROM RELEASE SHA256SUMS FILE!" ; done + + +extract_docker() { + image=$1 + docker_platform=$2 + restic_platform=$3 + out=restic_${restic_version}_linux_${restic_platform}.bz2 + + docker image pull --platform "linux/${docker_platform}" ${image}:${restic_version} > /dev/null + docker image save ${image}:${restic_version} -o docker.tar + + mkdir img + tar xvf docker.tar -C img --wildcards \*/layer.tar > /dev/null + rm docker.tar + for i in img/*/layer.tar; do + tar -xvf "$i" -C img usr/bin/restic 2> /dev/null 1>&2 || true + if [[ -f img/usr/bin/restic ]]; then + if [[ -f restic-docker ]]; then + set_invalid "WARNING: CONTAINER CONTAINS MULTIPLE RESTIC BINARIES" + fi + mv img/usr/bin/restic restic-docker + fi + done + + rm -rf img + bzip2 restic-docker + mv restic-docker.bz2 docker/${out} + grep ${out} SHA256SUMS >> docker/SHA256SUMS +} + +ctr=0 +for img in restic/restic ghcr.io/restic/restic; do + highlight "Verifying binaries in docker containers from $img" + mkdir docker + + extract_docker "$img" arm/v7 arm + extract_docker "$img" arm64 arm64 + extract_docker "$img" 386 386 + extract_docker "$img" amd64 amd64 + + (cd docker && shasum -a256 -c SHA256SUMS) || set_invalid "WARNING: DOCKER CONTAINER DOES NOT CONTAIN RELEASE BINARIES!" + + mv docker docker-$(( ctr++ )) +done + + +if [[ $is_valid -ne 1 ]]; then + highlight "Failed to reproduce some binaries, check the script output for details" + exit 1 +else + cd .. + rm -rf "${tmpdir}" +fi diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index 98819d797..e2f22ebea 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -267,7 +267,7 @@ func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi // FutureNode holds a reference to a channel that returns a FutureNodeResult // or a reference to an already existing result. If the result is available -// immediatelly, then storing a reference directly requires less memory than +// immediately, then storing a reference directly requires less memory than // using the indirection via a channel. type FutureNode struct { ch <-chan futureNodeResult diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index 3c87055d8..c6daed5bb 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -15,6 +15,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/errors" @@ -1842,26 +1843,26 @@ func TestArchiverErrorReporting(t *testing.T) { } type noCancelBackend struct { - restic.Backend + backend.Backend } -func (c *noCancelBackend) Remove(_ context.Context, h restic.Handle) error { +func (c *noCancelBackend) Remove(_ context.Context, h backend.Handle) error { return c.Backend.Remove(context.Background(), h) } -func (c *noCancelBackend) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) error { +func (c *noCancelBackend) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) error { return c.Backend.Save(context.Background(), h, rd) } -func (c *noCancelBackend) Load(_ context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (c *noCancelBackend) Load(_ context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { return c.Backend.Load(context.Background(), h, length, offset, fn) } -func (c *noCancelBackend) Stat(_ context.Context, h restic.Handle) (restic.FileInfo, error) { +func (c *noCancelBackend) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) { return c.Backend.Stat(context.Background(), h) } -func (c *noCancelBackend) List(_ context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (c *noCancelBackend) List(_ context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { return c.Backend.List(context.Background(), t, fn) } diff --git a/internal/archiver/file_saver.go b/internal/archiver/file_saver.go index 0742c8b57..724f5e620 100644 --- a/internal/archiver/file_saver.go +++ b/internal/archiver/file_saver.go @@ -2,6 +2,7 @@ package archiver import ( "context" + "fmt" "io" "os" "sync" @@ -146,7 +147,7 @@ func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat panic("completed twice") } isCompleted = true - fnr.err = err + fnr.err = fmt.Errorf("failed to save %v: %w", target, err) fnr.node = nil fnr.stats = ItemStats{} finish(fnr) diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index 50be63d5a..adaa37d97 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -15,9 +15,9 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" @@ -43,7 +43,7 @@ const saveLargeSize = 256 * 1024 * 1024 const defaultListMaxItems = 5000 // make sure that *Backend implements backend.Backend -var _ restic.Backend = &Backend{} +var _ backend.Backend = &Backend{} func NewFactory() location.Factory { return location.NewHTTPBackendFactory("azure", ParseConfig, location.NoPassword, Create, Open) @@ -197,7 +197,7 @@ func (be *Backend) Path() string { } // Save stores data in the backend at the handle. -func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { objName := be.Filename(h) debug.Log("InsertObject(%v, %v)", be.cfg.AccountName, objName) @@ -214,7 +214,7 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe return err } -func (be *Backend) saveSmall(ctx context.Context, objName string, rd restic.RewindReader) error { +func (be *Backend) saveSmall(ctx context.Context, objName string, rd backend.RewindReader) error { blockBlobClient := be.container.NewBlockBlobClient(objName) // upload it as a new "block", use the base64 hash for the ID @@ -239,7 +239,7 @@ func (be *Backend) saveSmall(ctx context.Context, objName string, rd restic.Rewi return errors.Wrap(err, "CommitBlockList") } -func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.RewindReader) error { +func (be *Backend) saveLarge(ctx context.Context, objName string, rd backend.RewindReader) error { blockBlobClient := be.container.NewBlockBlobClient(objName) buf := make([]byte, 100*1024*1024) @@ -294,11 +294,11 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn) +func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } -func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { objName := be.Filename(h) blockBlobClient := be.container.NewBlobClient(objName) @@ -317,17 +317,17 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, } // Stat returns information about a blob. -func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { objName := be.Filename(h) blobClient := be.container.NewBlobClient(objName) props, err := blobClient.GetProperties(ctx, nil) if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "blob.GetProperties") + return backend.FileInfo{}, errors.Wrap(err, "blob.GetProperties") } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Size: *props.ContentLength, Name: h.Name, } @@ -335,7 +335,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, } // Remove removes the blob with the given name and type. -func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (be *Backend) Remove(ctx context.Context, h backend.Handle) error { objName := be.Filename(h) blob := be.container.NewBlobClient(objName) @@ -350,7 +350,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { prefix, _ := be.Basedir(t) // make sure prefix ends with a slash @@ -381,7 +381,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F continue } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: path.Base(m), Size: *item.Properties.ContentLength, } @@ -407,7 +407,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F // Delete removes all restic keys in the bucket. It will not remove the bucket itself. func (be *Backend) Delete(ctx context.Context) error { - return backend.DefaultDelete(ctx, be) + return util.DefaultDelete(ctx, be) } // Close does nothing diff --git a/internal/backend/azure/azure_test.go b/internal/backend/azure/azure_test.go index 33f65bd52..7df27d325 100644 --- a/internal/backend/azure/azure_test.go +++ b/internal/backend/azure/azure_test.go @@ -122,11 +122,11 @@ func TestUploadLargeFile(t *testing.T) { data := rtest.Random(23, 300*1024*1024) id := restic.Hash(data) - h := restic.Handle{Name: id.String(), Type: restic.PackFile} + h := backend.Handle{Name: id.String(), Type: backend.PackFile} t.Logf("hash of %d bytes: %v", len(data), id) - err = be.Save(ctx, h, restic.NewByteReader(data, be.Hasher())) + err = be.Save(ctx, h, backend.NewByteReader(data, be.Hasher())) if err != nil { t.Fatal(err) } diff --git a/internal/backend/azure/config.go b/internal/backend/azure/config.go index d819b35aa..6ae431f65 100644 --- a/internal/backend/azure/config.go +++ b/internal/backend/azure/config.go @@ -5,9 +5,9 @@ import ( "path" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" ) // Config contains all configuration necessary to connect to an azure compatible @@ -57,7 +57,7 @@ func ParseConfig(s string) (*Config, error) { return &cfg, nil } -var _ restic.ApplyEnvironmenter = &Config{} +var _ backend.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. func (cfg *Config) ApplyEnvironment(prefix string) { diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go index 0bd3b994c..bc6ef1a4d 100644 --- a/internal/backend/b2/b2.go +++ b/internal/backend/b2/b2.go @@ -12,12 +12,12 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" - "github.com/kurin/blazer/b2" - "github.com/kurin/blazer/base" + "github.com/Backblaze/blazer/b2" + "github.com/Backblaze/blazer/base" ) // b2Backend is a backend which stores its data on Backblaze B2. @@ -31,11 +31,11 @@ type b2Backend struct { canDelete bool } -// Billing happens in 1000 item granlarity, but we are more interested in reducing the number of network round trips +// Billing happens in 1000 item granularity, but we are more interested in reducing the number of network round trips const defaultListMaxItems = 10 * 1000 -// ensure statically that *b2Backend implements restic.Backend. -var _ restic.Backend = &b2Backend{} +// ensure statically that *b2Backend implements backend.Backend. +var _ backend.Backend = &b2Backend{} func NewFactory() location.Factory { return location.NewHTTPBackendFactory("b2", ParseConfig, location.NoPassword, Create, Open) @@ -85,7 +85,7 @@ func newClient(ctx context.Context, cfg Config, rt http.RoundTripper) (*b2.Clien } // Open opens a connection to the B2 service. -func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { debug.Log("cfg %#v", cfg) ctx, cancel := context.WithCancel(ctx) @@ -118,7 +118,7 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend // Create opens a connection to the B2 service. If the bucket does not exist yet, // it is created. -func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { debug.Log("cfg %#v", cfg) ctx, cancel := context.WithCancel(ctx) @@ -188,14 +188,14 @@ func (be *b2Backend) IsNotExist(err error) bool { // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *b2Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *b2Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { ctx, cancel := context.WithCancel(ctx) defer cancel() - return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn) + return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } -func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *b2Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { name := be.Layout.Filename(h) obj := be.bucket.Object(name) @@ -213,7 +213,7 @@ func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int } // Save stores data in the backend at the handle. -func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *b2Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -237,18 +237,18 @@ func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.Rewind } // Stat returns information about a blob. -func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { +func (be *b2Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) { name := be.Filename(h) obj := be.bucket.Object(name) info, err := obj.Attrs(ctx) if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "Stat") + return backend.FileInfo{}, errors.Wrap(err, "Stat") } - return restic.FileInfo{Size: info.Size, Name: h.Name}, nil + return backend.FileInfo{Size: info.Size, Name: h.Name}, nil } // Remove removes the blob with the given name and type. -func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error { +func (be *b2Backend) Remove(ctx context.Context, h backend.Handle) error { // the retry backend will also repeat the remove method up to 10 times for i := 0; i < 3; i++ { obj := be.bucket.Object(be.Filename(h)) @@ -284,7 +284,7 @@ func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error { } // List returns a channel that yields all names of blobs of type t. -func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *b2Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -299,7 +299,7 @@ func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic return err } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: path.Base(obj.Name()), Size: attrs.Size, } @@ -313,7 +313,7 @@ func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic // Delete removes all restic keys in the bucket. It will not remove the bucket itself. func (be *b2Backend) Delete(ctx context.Context) error { - return backend.DefaultDelete(ctx, be) + return util.DefaultDelete(ctx, be) } // Close does nothing diff --git a/internal/backend/b2/config.go b/internal/backend/b2/config.go index 94614e44f..8d947fc1b 100644 --- a/internal/backend/b2/config.go +++ b/internal/backend/b2/config.go @@ -6,9 +6,9 @@ import ( "regexp" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" ) // Config contains all configuration necessary to connect to an b2 compatible @@ -82,7 +82,7 @@ func ParseConfig(s string) (*Config, error) { return &cfg, nil } -var _ restic.ApplyEnvironmenter = &Config{} +var _ backend.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. func (cfg *Config) ApplyEnvironment(prefix string) { diff --git a/internal/restic/backend.go b/internal/backend/backend.go similarity index 79% rename from internal/restic/backend.go rename to internal/backend/backend.go index 555b9d96e..aa9920f9b 100644 --- a/internal/restic/backend.go +++ b/internal/backend/backend.go @@ -1,4 +1,4 @@ -package restic +package backend import ( "context" @@ -18,7 +18,7 @@ type Backend interface { // repository. Location() string - // Connections returns the maxmimum number of concurrent backend operations. + // Connections returns the maximum number of concurrent backend operations. Connections() uint // Hasher may return a hash function for calculating a content hash for the backend @@ -27,7 +27,7 @@ type Backend interface { // HasAtomicReplace returns whether Save() can atomically replace files HasAtomicReplace() bool - // Remove removes a File described by h. + // Remove removes a File described by h. Remove(ctx context.Context, h Handle) error // Close the backend @@ -43,7 +43,7 @@ type Backend interface { // The function fn may be called multiple times during the same Load invocation // and therefore must be idempotent. // - // Implementations are encouraged to use backend.DefaultLoad + // Implementations are encouraged to use util.DefaultLoad Load(ctx context.Context, h Handle, length int, offset int64, fn func(rd io.Reader) error) error // Stat returns information about the File identified by h. @@ -70,11 +70,36 @@ type Backend interface { Delete(ctx context.Context) error } -type BackendUnwrapper interface { +type Unwrapper interface { // Unwrap returns the underlying backend or nil if there is none. Unwrap() Backend } +func AsBackend[B Backend](b Backend) B { + for b != nil { + if be, ok := b.(B); ok { + return be + } + + if be, ok := b.(Unwrapper); ok { + b = be.Unwrap() + } else { + // not the backend we're looking for + break + } + } + var be B + return be +} + +type FreezeBackend interface { + Backend + // Freeze blocks all backend operations except those on lock files + Freeze() + // Unfreeze allows all backend operations to continue + Unfreeze() +} + // FileInfo is contains information about a file in the backend. type FileInfo struct { Size int64 diff --git a/internal/backend/backend_test.go b/internal/backend/backend_test.go new file mode 100644 index 000000000..28ece55df --- /dev/null +++ b/internal/backend/backend_test.go @@ -0,0 +1,38 @@ +package backend_test + +import ( + "testing" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/test" +) + +type testBackend struct { + backend.Backend +} + +func (t *testBackend) Unwrap() backend.Backend { + return nil +} + +type otherTestBackend struct { + backend.Backend +} + +func (t *otherTestBackend) Unwrap() backend.Backend { + return t.Backend +} + +func TestAsBackend(t *testing.T) { + other := otherTestBackend{} + test.Assert(t, backend.AsBackend[*testBackend](other) == nil, "otherTestBackend is not a testBackend backend") + + testBe := &testBackend{} + test.Assert(t, backend.AsBackend[*testBackend](testBe) == testBe, "testBackend was not returned") + + wrapper := &otherTestBackend{Backend: testBe} + test.Assert(t, backend.AsBackend[*testBackend](wrapper) == testBe, "failed to unwrap testBackend backend") + + wrapper.Backend = other + test.Assert(t, backend.AsBackend[*testBackend](wrapper) == nil, "a wrapped otherTestBackend is not a testBackend") +} diff --git a/internal/backend/dryrun/dry_backend.go b/internal/backend/dryrun/dry_backend.go index f7acb10dd..b3db0210f 100644 --- a/internal/backend/dryrun/dry_backend.go +++ b/internal/backend/dryrun/dry_backend.go @@ -5,8 +5,8 @@ import ( "hash" "io" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/restic" ) // Backend passes reads through to an underlying layer and accepts writes, but @@ -15,20 +15,20 @@ import ( // the repo and does normal operations else. // This is used for `backup --dry-run`. type Backend struct { - b restic.Backend + b backend.Backend } -// statically ensure that Backend implements restic.Backend. -var _ restic.Backend = &Backend{} +// statically ensure that Backend implements backend.Backend. +var _ backend.Backend = &Backend{} -func New(be restic.Backend) *Backend { +func New(be backend.Backend) *Backend { b := &Backend{b: be} debug.Log("created new dry backend") return b } // Save adds new Data to the backend. -func (be *Backend) Save(_ context.Context, h restic.Handle, _ restic.RewindReader) error { +func (be *Backend) Save(_ context.Context, h backend.Handle, _ backend.RewindReader) error { if err := h.Valid(); err != nil { return err } @@ -38,7 +38,7 @@ func (be *Backend) Save(_ context.Context, h restic.Handle, _ restic.RewindReade } // Remove deletes a file from the backend. -func (be *Backend) Remove(_ context.Context, _ restic.Handle) error { +func (be *Backend) Remove(_ context.Context, _ backend.Handle) error { return nil } @@ -72,14 +72,14 @@ func (be *Backend) IsNotExist(err error) bool { return be.b.IsNotExist(err) } -func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { return be.b.List(ctx, t, fn) } -func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(io.Reader) error) error { +func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(io.Reader) error) error { return be.b.Load(ctx, h, length, offset, fn) } -func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { return be.b.Stat(ctx, h) } diff --git a/internal/backend/dryrun/dry_backend_test.go b/internal/backend/dryrun/dry_backend_test.go index 69716c340..56962107d 100644 --- a/internal/backend/dryrun/dry_backend_test.go +++ b/internal/backend/dryrun/dry_backend_test.go @@ -8,16 +8,16 @@ import ( "strings" "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/dryrun" "github.com/restic/restic/internal/backend/mem" ) // make sure that Backend implements backend.Backend -var _ restic.Backend = &dryrun.Backend{} +var _ backend.Backend = &dryrun.Backend{} -func newBackends() (*dryrun.Backend, restic.Backend) { +func newBackends() (*dryrun.Backend, backend.Backend) { m := mem.New() return dryrun.New(m), m } @@ -30,7 +30,7 @@ func TestDry(t *testing.T) { // won't pass. Instead, perform a series of operations over the backend, testing the state // at each step. steps := []struct { - be restic.Backend + be backend.Backend op string fname string content string @@ -61,13 +61,13 @@ func TestDry(t *testing.T) { for i, step := range steps { var err error - handle := restic.Handle{Type: restic.PackFile, Name: step.fname} + handle := backend.Handle{Type: backend.PackFile, Name: step.fname} switch step.op { case "save": - err = step.be.Save(ctx, handle, restic.NewByteReader([]byte(step.content), step.be.Hasher())) + err = step.be.Save(ctx, handle, backend.NewByteReader([]byte(step.content), step.be.Hasher())) case "list": fileList := []string{} - err = step.be.List(ctx, restic.PackFile, func(fi restic.FileInfo) error { + err = step.be.List(ctx, backend.PackFile, func(fi backend.FileInfo) error { fileList = append(fileList, fi.Name) return nil }) @@ -86,7 +86,7 @@ func TestDry(t *testing.T) { case "remove": err = step.be.Remove(ctx, handle) case "stat": - var fi restic.FileInfo + var fi backend.FileInfo fi, err = step.be.Stat(ctx, handle) if err == nil { fis := fmt.Sprintf("%s %d", fi.Name, fi.Size) diff --git a/internal/restic/file.go b/internal/backend/file.go similarity index 92% rename from internal/restic/file.go rename to internal/backend/file.go index 0e9f046ae..990175f9c 100644 --- a/internal/restic/file.go +++ b/internal/backend/file.go @@ -1,4 +1,4 @@ -package restic +package backend import ( "fmt" @@ -41,9 +41,9 @@ func (t FileType) String() string { // Handle is used to store and access data in a backend. type Handle struct { - Type FileType - ContainedBlobType BlobType - Name string + Type FileType + IsMetadata bool + Name string } func (h Handle) String() string { diff --git a/internal/restic/file_test.go b/internal/backend/file_test.go similarity index 98% rename from internal/restic/file_test.go rename to internal/backend/file_test.go index cc54c2924..45f1c2ee7 100644 --- a/internal/restic/file_test.go +++ b/internal/backend/file_test.go @@ -1,4 +1,4 @@ -package restic +package backend import ( "testing" diff --git a/internal/backend/gs/config.go b/internal/backend/gs/config.go index 61a31113f..7dc181ce9 100644 --- a/internal/backend/gs/config.go +++ b/internal/backend/gs/config.go @@ -5,9 +5,9 @@ import ( "path" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" ) // Config contains all configuration necessary to connect to a Google Cloud Storage @@ -59,7 +59,7 @@ func ParseConfig(s string) (*Config, error) { return &cfg, nil } -var _ restic.ApplyEnvironmenter = &Config{} +var _ backend.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. func (cfg *Config) ApplyEnvironment(prefix string) { diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index 5c12654d6..77d20e056 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -16,8 +16,8 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/restic" "golang.org/x/oauth2" "golang.org/x/oauth2/google" @@ -45,8 +45,8 @@ type Backend struct { layout.Layout } -// Ensure that *Backend implements restic.Backend. -var _ restic.Backend = &Backend{} +// Ensure that *Backend implements backend.Backend. +var _ backend.Backend = &Backend{} func NewFactory() location.Factory { return location.NewHTTPBackendFactory("gs", ParseConfig, location.NoPassword, Create, Open) @@ -122,7 +122,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { } // Open opens the gs backend at the specified bucket. -func Open(_ context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Open(_ context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { return open(cfg, rt) } @@ -131,7 +131,7 @@ func Open(_ context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, // // The service account must have the "storage.buckets.create" permission to // create a bucket the does not yet exist. -func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { be, err := open(cfg, rt) if err != nil { return nil, errors.Wrap(err, "open") @@ -203,7 +203,7 @@ func (be *Backend) Path() string { } // Save stores data in the backend at the handle. -func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { objName := be.Filename(h) // Set chunk size to zero to disable resumable uploads. @@ -253,14 +253,14 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { ctx, cancel := context.WithCancel(ctx) defer cancel() - return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn) + return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } -func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { if length == 0 { // negative length indicates read till end to GCS lib length = -1 @@ -277,20 +277,20 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, } // Stat returns information about a blob. -func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { +func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) { objName := be.Filename(h) attr, err := be.bucket.Object(objName).Attrs(ctx) if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "service.Objects.Get") + return backend.FileInfo{}, errors.Wrap(err, "service.Objects.Get") } - return restic.FileInfo{Size: attr.Size, Name: h.Name}, nil + return backend.FileInfo{Size: attr.Size, Name: h.Name}, nil } // Remove removes the blob with the given name and type. -func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (be *Backend) Remove(ctx context.Context, h backend.Handle) error { objName := be.Filename(h) err := be.bucket.Object(objName).Delete(ctx) @@ -304,7 +304,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { prefix, _ := be.Basedir(t) // make sure prefix ends with a slash @@ -330,7 +330,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F continue } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: path.Base(m), Size: int64(attrs.Size), } @@ -350,7 +350,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F // Delete removes all restic keys in the bucket. It will not remove the bucket itself. func (be *Backend) Delete(ctx context.Context) error { - return backend.DefaultDelete(ctx, be) + return util.DefaultDelete(ctx, be) } // Close does nothing. diff --git a/internal/backend/layout/layout.go b/internal/backend/layout/layout.go index b83f4c05b..b600566a4 100644 --- a/internal/backend/layout/layout.go +++ b/internal/backend/layout/layout.go @@ -7,6 +7,7 @@ import ( "path/filepath" "regexp" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" @@ -15,9 +16,9 @@ import ( // Layout computes paths for file name storage. type Layout interface { - Filename(restic.Handle) string - Dirname(restic.Handle) string - Basedir(restic.FileType) (dir string, subdirs bool) + Filename(backend.Handle) string + Dirname(backend.Handle) string + Basedir(backend.FileType) (dir string, subdirs bool) Paths() []string Name() string } @@ -102,13 +103,13 @@ func DetectLayout(ctx context.Context, repo Filesystem, dir string) (Layout, err } // key file in the "keys" dir (DefaultLayout) - foundKeysFile, err := hasBackendFile(ctx, repo, repo.Join(dir, defaultLayoutPaths[restic.KeyFile])) + foundKeysFile, err := hasBackendFile(ctx, repo, repo.Join(dir, defaultLayoutPaths[backend.KeyFile])) if err != nil { return nil, err } // key file in the "key" dir (S3LegacyLayout) - foundKeyFile, err := hasBackendFile(ctx, repo, repo.Join(dir, s3LayoutPaths[restic.KeyFile])) + foundKeyFile, err := hasBackendFile(ctx, repo, repo.Join(dir, s3LayoutPaths[backend.KeyFile])) if err != nil { return nil, err } diff --git a/internal/backend/layout/layout_default.go b/internal/backend/layout/layout_default.go index 17c250e8f..9a8419f10 100644 --- a/internal/backend/layout/layout_default.go +++ b/internal/backend/layout/layout_default.go @@ -3,7 +3,7 @@ package layout import ( "encoding/hex" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" ) // DefaultLayout implements the default layout for local and sftp backends, as @@ -15,12 +15,12 @@ type DefaultLayout struct { Join func(...string) string } -var defaultLayoutPaths = map[restic.FileType]string{ - restic.PackFile: "data", - restic.SnapshotFile: "snapshots", - restic.IndexFile: "index", - restic.LockFile: "locks", - restic.KeyFile: "keys", +var defaultLayoutPaths = map[backend.FileType]string{ + backend.PackFile: "data", + backend.SnapshotFile: "snapshots", + backend.IndexFile: "index", + backend.LockFile: "locks", + backend.KeyFile: "keys", } func (l *DefaultLayout) String() string { @@ -33,10 +33,10 @@ func (l *DefaultLayout) Name() string { } // Dirname returns the directory path for a given file type and name. -func (l *DefaultLayout) Dirname(h restic.Handle) string { +func (l *DefaultLayout) Dirname(h backend.Handle) string { p := defaultLayoutPaths[h.Type] - if h.Type == restic.PackFile && len(h.Name) > 2 { + if h.Type == backend.PackFile && len(h.Name) > 2 { p = l.Join(p, h.Name[:2]) + "/" } @@ -44,9 +44,9 @@ func (l *DefaultLayout) Dirname(h restic.Handle) string { } // Filename returns a path to a file, including its name. -func (l *DefaultLayout) Filename(h restic.Handle) string { +func (l *DefaultLayout) Filename(h backend.Handle) string { name := h.Name - if h.Type == restic.ConfigFile { + if h.Type == backend.ConfigFile { return l.Join(l.Path, "config") } @@ -62,15 +62,15 @@ func (l *DefaultLayout) Paths() (dirs []string) { // also add subdirs for i := 0; i < 256; i++ { subdir := hex.EncodeToString([]byte{byte(i)}) - dirs = append(dirs, l.Join(l.Path, defaultLayoutPaths[restic.PackFile], subdir)) + dirs = append(dirs, l.Join(l.Path, defaultLayoutPaths[backend.PackFile], subdir)) } return dirs } // Basedir returns the base dir name for type t. -func (l *DefaultLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) { - if t == restic.PackFile { +func (l *DefaultLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) { + if t == backend.PackFile { subdirs = true } diff --git a/internal/backend/layout/layout_rest.go b/internal/backend/layout/layout_rest.go index 2aa869995..822dd4a7e 100644 --- a/internal/backend/layout/layout_rest.go +++ b/internal/backend/layout/layout_rest.go @@ -1,6 +1,8 @@ package layout -import "github.com/restic/restic/internal/restic" +import ( + "github.com/restic/restic/internal/backend" +) // RESTLayout implements the default layout for the REST protocol. type RESTLayout struct { @@ -21,8 +23,8 @@ func (l *RESTLayout) Name() string { } // Dirname returns the directory path for a given file type and name. -func (l *RESTLayout) Dirname(h restic.Handle) string { - if h.Type == restic.ConfigFile { +func (l *RESTLayout) Dirname(h backend.Handle) string { + if h.Type == backend.ConfigFile { return l.URL + l.Join(l.Path, "/") } @@ -30,10 +32,10 @@ func (l *RESTLayout) Dirname(h restic.Handle) string { } // Filename returns a path to a file, including its name. -func (l *RESTLayout) Filename(h restic.Handle) string { +func (l *RESTLayout) Filename(h backend.Handle) string { name := h.Name - if h.Type == restic.ConfigFile { + if h.Type == backend.ConfigFile { name = "config" } @@ -49,6 +51,6 @@ func (l *RESTLayout) Paths() (dirs []string) { } // Basedir returns the base dir name for files of type t. -func (l *RESTLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) { +func (l *RESTLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) { return l.URL + l.Join(l.Path, restLayoutPaths[t]), false } diff --git a/internal/backend/layout/layout_s3legacy.go b/internal/backend/layout/layout_s3legacy.go index ac88e77ad..8b90789d8 100644 --- a/internal/backend/layout/layout_s3legacy.go +++ b/internal/backend/layout/layout_s3legacy.go @@ -1,6 +1,8 @@ package layout -import "github.com/restic/restic/internal/restic" +import ( + "github.com/restic/restic/internal/backend" +) // S3LegacyLayout implements the old layout used for s3 cloud storage backends, as // described in the Design document. @@ -10,12 +12,12 @@ type S3LegacyLayout struct { Join func(...string) string } -var s3LayoutPaths = map[restic.FileType]string{ - restic.PackFile: "data", - restic.SnapshotFile: "snapshot", - restic.IndexFile: "index", - restic.LockFile: "lock", - restic.KeyFile: "key", +var s3LayoutPaths = map[backend.FileType]string{ + backend.PackFile: "data", + backend.SnapshotFile: "snapshot", + backend.IndexFile: "index", + backend.LockFile: "lock", + backend.KeyFile: "key", } func (l *S3LegacyLayout) String() string { @@ -44,8 +46,8 @@ func (l *S3LegacyLayout) join(url string, items ...string) string { } // Dirname returns the directory path for a given file type and name. -func (l *S3LegacyLayout) Dirname(h restic.Handle) string { - if h.Type == restic.ConfigFile { +func (l *S3LegacyLayout) Dirname(h backend.Handle) string { + if h.Type == backend.ConfigFile { return l.URL + l.Join(l.Path, "/") } @@ -53,10 +55,10 @@ func (l *S3LegacyLayout) Dirname(h restic.Handle) string { } // Filename returns a path to a file, including its name. -func (l *S3LegacyLayout) Filename(h restic.Handle) string { +func (l *S3LegacyLayout) Filename(h backend.Handle) string { name := h.Name - if h.Type == restic.ConfigFile { + if h.Type == backend.ConfigFile { name = "config" } @@ -72,6 +74,6 @@ func (l *S3LegacyLayout) Paths() (dirs []string) { } // Basedir returns the base dir name for type t. -func (l *S3LegacyLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) { +func (l *S3LegacyLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) { return l.Join(l.Path, s3LayoutPaths[t]), false } diff --git a/internal/backend/layout/layout_test.go b/internal/backend/layout/layout_test.go index fc9c6e214..998f5aeb6 100644 --- a/internal/backend/layout/layout_test.go +++ b/internal/backend/layout/layout_test.go @@ -9,7 +9,7 @@ import ( "sort" "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" rtest "github.com/restic/restic/internal/test" ) @@ -19,79 +19,79 @@ func TestDefaultLayout(t *testing.T) { var tests = []struct { path string join func(...string) string - restic.Handle + backend.Handle filename string }{ { tempdir, filepath.Join, - restic.Handle{Type: restic.PackFile, Name: "0123456"}, + backend.Handle{Type: backend.PackFile, Name: "0123456"}, filepath.Join(tempdir, "data", "01", "0123456"), }, { tempdir, filepath.Join, - restic.Handle{Type: restic.ConfigFile, Name: "CFG"}, + backend.Handle{Type: backend.ConfigFile, Name: "CFG"}, filepath.Join(tempdir, "config"), }, { tempdir, filepath.Join, - restic.Handle{Type: restic.SnapshotFile, Name: "123456"}, + backend.Handle{Type: backend.SnapshotFile, Name: "123456"}, filepath.Join(tempdir, "snapshots", "123456"), }, { tempdir, filepath.Join, - restic.Handle{Type: restic.IndexFile, Name: "123456"}, + backend.Handle{Type: backend.IndexFile, Name: "123456"}, filepath.Join(tempdir, "index", "123456"), }, { tempdir, filepath.Join, - restic.Handle{Type: restic.LockFile, Name: "123456"}, + backend.Handle{Type: backend.LockFile, Name: "123456"}, filepath.Join(tempdir, "locks", "123456"), }, { tempdir, filepath.Join, - restic.Handle{Type: restic.KeyFile, Name: "123456"}, + backend.Handle{Type: backend.KeyFile, Name: "123456"}, filepath.Join(tempdir, "keys", "123456"), }, { "", path.Join, - restic.Handle{Type: restic.PackFile, Name: "0123456"}, + backend.Handle{Type: backend.PackFile, Name: "0123456"}, "data/01/0123456", }, { "", path.Join, - restic.Handle{Type: restic.ConfigFile, Name: "CFG"}, + backend.Handle{Type: backend.ConfigFile, Name: "CFG"}, "config", }, { "", path.Join, - restic.Handle{Type: restic.SnapshotFile, Name: "123456"}, + backend.Handle{Type: backend.SnapshotFile, Name: "123456"}, "snapshots/123456", }, { "", path.Join, - restic.Handle{Type: restic.IndexFile, Name: "123456"}, + backend.Handle{Type: backend.IndexFile, Name: "123456"}, "index/123456", }, { "", path.Join, - restic.Handle{Type: restic.LockFile, Name: "123456"}, + backend.Handle{Type: backend.LockFile, Name: "123456"}, "locks/123456", }, { "", path.Join, - restic.Handle{Type: restic.KeyFile, Name: "123456"}, + backend.Handle{Type: backend.KeyFile, Name: "123456"}, "keys/123456", }, } @@ -143,31 +143,31 @@ func TestRESTLayout(t *testing.T) { path := rtest.TempDir(t) var tests = []struct { - restic.Handle + backend.Handle filename string }{ { - restic.Handle{Type: restic.PackFile, Name: "0123456"}, + backend.Handle{Type: backend.PackFile, Name: "0123456"}, filepath.Join(path, "data", "0123456"), }, { - restic.Handle{Type: restic.ConfigFile, Name: "CFG"}, + backend.Handle{Type: backend.ConfigFile, Name: "CFG"}, filepath.Join(path, "config"), }, { - restic.Handle{Type: restic.SnapshotFile, Name: "123456"}, + backend.Handle{Type: backend.SnapshotFile, Name: "123456"}, filepath.Join(path, "snapshots", "123456"), }, { - restic.Handle{Type: restic.IndexFile, Name: "123456"}, + backend.Handle{Type: backend.IndexFile, Name: "123456"}, filepath.Join(path, "index", "123456"), }, { - restic.Handle{Type: restic.LockFile, Name: "123456"}, + backend.Handle{Type: backend.LockFile, Name: "123456"}, filepath.Join(path, "locks", "123456"), }, { - restic.Handle{Type: restic.KeyFile, Name: "123456"}, + backend.Handle{Type: backend.KeyFile, Name: "123456"}, filepath.Join(path, "keys", "123456"), }, } @@ -209,61 +209,61 @@ func TestRESTLayout(t *testing.T) { func TestRESTLayoutURLs(t *testing.T) { var tests = []struct { l Layout - h restic.Handle + h backend.Handle fn string dir string }{ { &RESTLayout{URL: "https://hostname.foo", Path: "", Join: path.Join}, - restic.Handle{Type: restic.PackFile, Name: "foobar"}, + backend.Handle{Type: backend.PackFile, Name: "foobar"}, "https://hostname.foo/data/foobar", "https://hostname.foo/data/", }, { &RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, - restic.Handle{Type: restic.LockFile, Name: "foobar"}, + backend.Handle{Type: backend.LockFile, Name: "foobar"}, "https://hostname.foo:1234/prefix/repo/locks/foobar", "https://hostname.foo:1234/prefix/repo/locks/", }, { &RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, - restic.Handle{Type: restic.ConfigFile, Name: "foobar"}, + backend.Handle{Type: backend.ConfigFile, Name: "foobar"}, "https://hostname.foo:1234/prefix/repo/config", "https://hostname.foo:1234/prefix/repo/", }, { &S3LegacyLayout{URL: "https://hostname.foo", Path: "/", Join: path.Join}, - restic.Handle{Type: restic.PackFile, Name: "foobar"}, + backend.Handle{Type: backend.PackFile, Name: "foobar"}, "https://hostname.foo/data/foobar", "https://hostname.foo/data/", }, { &S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join}, - restic.Handle{Type: restic.LockFile, Name: "foobar"}, + backend.Handle{Type: backend.LockFile, Name: "foobar"}, "https://hostname.foo:1234/prefix/repo/lock/foobar", "https://hostname.foo:1234/prefix/repo/lock/", }, { &S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, - restic.Handle{Type: restic.ConfigFile, Name: "foobar"}, + backend.Handle{Type: backend.ConfigFile, Name: "foobar"}, "https://hostname.foo:1234/prefix/repo/config", "https://hostname.foo:1234/prefix/repo/", }, { &S3LegacyLayout{URL: "", Path: "", Join: path.Join}, - restic.Handle{Type: restic.PackFile, Name: "foobar"}, + backend.Handle{Type: backend.PackFile, Name: "foobar"}, "data/foobar", "data/", }, { &S3LegacyLayout{URL: "", Path: "", Join: path.Join}, - restic.Handle{Type: restic.LockFile, Name: "foobar"}, + backend.Handle{Type: backend.LockFile, Name: "foobar"}, "lock/foobar", "lock/", }, { &S3LegacyLayout{URL: "", Path: "/", Join: path.Join}, - restic.Handle{Type: restic.ConfigFile, Name: "foobar"}, + backend.Handle{Type: backend.ConfigFile, Name: "foobar"}, "/config", "/", }, @@ -288,31 +288,31 @@ func TestS3LegacyLayout(t *testing.T) { path := rtest.TempDir(t) var tests = []struct { - restic.Handle + backend.Handle filename string }{ { - restic.Handle{Type: restic.PackFile, Name: "0123456"}, + backend.Handle{Type: backend.PackFile, Name: "0123456"}, filepath.Join(path, "data", "0123456"), }, { - restic.Handle{Type: restic.ConfigFile, Name: "CFG"}, + backend.Handle{Type: backend.ConfigFile, Name: "CFG"}, filepath.Join(path, "config"), }, { - restic.Handle{Type: restic.SnapshotFile, Name: "123456"}, + backend.Handle{Type: backend.SnapshotFile, Name: "123456"}, filepath.Join(path, "snapshot", "123456"), }, { - restic.Handle{Type: restic.IndexFile, Name: "123456"}, + backend.Handle{Type: backend.IndexFile, Name: "123456"}, filepath.Join(path, "index", "123456"), }, { - restic.Handle{Type: restic.LockFile, Name: "123456"}, + backend.Handle{Type: backend.LockFile, Name: "123456"}, filepath.Join(path, "lock", "123456"), }, { - restic.Handle{Type: restic.KeyFile, Name: "123456"}, + backend.Handle{Type: backend.KeyFile, Name: "123456"}, filepath.Join(path, "key", "123456"), }, } @@ -415,8 +415,8 @@ func TestParseLayout(t *testing.T) { } // test that the functions work (and don't panic) - _ = layout.Dirname(restic.Handle{Type: restic.PackFile}) - _ = layout.Filename(restic.Handle{Type: restic.PackFile, Name: "1234"}) + _ = layout.Dirname(backend.Handle{Type: backend.PackFile}) + _ = layout.Filename(backend.Handle{Type: backend.PackFile, Name: "1234"}) _ = layout.Paths() layoutName := fmt.Sprintf("%T", layout) diff --git a/internal/backend/limiter/limiter.go b/internal/backend/limiter/limiter.go index 8cbe297fe..7ba5ad02b 100644 --- a/internal/backend/limiter/limiter.go +++ b/internal/backend/limiter/limiter.go @@ -5,8 +5,8 @@ import ( "net/http" ) -// Limiter defines an interface that implementors can use to rate limit I/O -// according to some policy defined and configured by the implementor. +// Limiter defines an interface that implementers can use to rate limit I/O +// according to some policy defined and configured by the implementer. type Limiter interface { // Upstream returns a rate limited reader that is intended to be used in // uploads. diff --git a/internal/backend/limiter/limiter_backend.go b/internal/backend/limiter/limiter_backend.go index a91794037..ac1a4188a 100644 --- a/internal/backend/limiter/limiter_backend.go +++ b/internal/backend/limiter/limiter_backend.go @@ -4,12 +4,12 @@ import ( "context" "io" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" ) -func WrapBackendConstructor[B restic.Backend, C any](constructor func(ctx context.Context, cfg C) (B, error)) func(ctx context.Context, cfg C, lim Limiter) (restic.Backend, error) { - return func(ctx context.Context, cfg C, lim Limiter) (restic.Backend, error) { - var be restic.Backend +func WrapBackendConstructor[B backend.Backend, C any](constructor func(ctx context.Context, cfg C) (B, error)) func(ctx context.Context, cfg C, lim Limiter) (backend.Backend, error) { + return func(ctx context.Context, cfg C, lim Limiter) (backend.Backend, error) { + var be backend.Backend be, err := constructor(ctx, cfg) if err != nil { return nil, err @@ -24,7 +24,7 @@ func WrapBackendConstructor[B restic.Backend, C any](constructor func(ctx contex // LimitBackend wraps a Backend and applies rate limiting to Load() and Save() // calls on the backend. -func LimitBackend(be restic.Backend, l Limiter) restic.Backend { +func LimitBackend(be backend.Backend, l Limiter) backend.Backend { return rateLimitedBackend{ Backend: be, limiter: l, @@ -32,11 +32,11 @@ func LimitBackend(be restic.Backend, l Limiter) restic.Backend { } type rateLimitedBackend struct { - restic.Backend + backend.Backend limiter Limiter } -func (r rateLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (r rateLimitedBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { limited := limitedRewindReader{ RewindReader: rd, limited: r.limiter.Upstream(rd), @@ -46,7 +46,7 @@ func (r rateLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic } type limitedRewindReader struct { - restic.RewindReader + backend.RewindReader limited io.Reader } @@ -55,13 +55,13 @@ func (l limitedRewindReader) Read(b []byte) (int, error) { return l.limited.Read(b) } -func (r rateLimitedBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { +func (r rateLimitedBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { return r.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error { return consumer(newDownstreamLimitedReader(rd, r.limiter)) }) } -func (r rateLimitedBackend) Unwrap() restic.Backend { return r.Backend } +func (r rateLimitedBackend) Unwrap() backend.Backend { return r.Backend } type limitedReader struct { io.Reader @@ -85,4 +85,4 @@ func (l *limitedReader) WriteTo(w io.Writer) (int64, error) { return l.writerTo.WriteTo(l.limiter.DownstreamWriter(w)) } -var _ restic.Backend = (*rateLimitedBackend)(nil) +var _ backend.Backend = (*rateLimitedBackend)(nil) diff --git a/internal/backend/limiter/limiter_backend_test.go b/internal/backend/limiter/limiter_backend_test.go index 1014dbed1..491d2ef69 100644 --- a/internal/backend/limiter/limiter_backend_test.go +++ b/internal/backend/limiter/limiter_backend_test.go @@ -8,8 +8,8 @@ import ( "io" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mock" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -21,11 +21,11 @@ func randomBytes(t *testing.T, size int) []byte { } func TestLimitBackendSave(t *testing.T) { - testHandle := restic.Handle{Type: restic.PackFile, Name: "test"} + testHandle := backend.Handle{Type: backend.PackFile, Name: "test"} data := randomBytes(t, 1234) be := mock.NewBackend() - be.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + be.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { buf := new(bytes.Buffer) _, err := io.Copy(buf, rd) if err != nil { @@ -39,7 +39,7 @@ func TestLimitBackendSave(t *testing.T) { limiter := NewStaticLimiter(Limits{42 * 1024, 42 * 1024}) limbe := LimitBackend(be, limiter) - rd := restic.NewByteReader(data, nil) + rd := backend.NewByteReader(data, nil) err := limbe.Save(context.TODO(), testHandle, rd) rtest.OK(t, err) } @@ -64,7 +64,7 @@ func (r *tracedReadWriteToCloser) Close() error { } func TestLimitBackendLoad(t *testing.T) { - testHandle := restic.Handle{Type: restic.PackFile, Name: "test"} + testHandle := backend.Handle{Type: backend.PackFile, Name: "test"} data := randomBytes(t, 1234) for _, test := range []struct { @@ -72,7 +72,7 @@ func TestLimitBackendLoad(t *testing.T) { }{{false, false}, {false, true}, {true, false}, {true, true}} { be := mock.NewBackend() src := newTracedReadWriteToCloser(bytes.NewReader(data)) - be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { if length != 0 || offset != 0 { return nil, fmt.Errorf("Not supported") } diff --git a/internal/backend/limiter/static_limiter.go b/internal/backend/limiter/static_limiter.go index 9fb8fbf24..c09a12de0 100644 --- a/internal/backend/limiter/static_limiter.go +++ b/internal/backend/limiter/static_limiter.go @@ -1,15 +1,16 @@ package limiter import ( + "context" "io" "net/http" - "github.com/juju/ratelimit" + "golang.org/x/time/rate" ) type staticLimiter struct { - upstream *ratelimit.Bucket - downstream *ratelimit.Bucket + upstream *rate.Limiter + downstream *rate.Limiter } // Limits represents static upload and download limits. @@ -23,16 +24,16 @@ type Limits struct { // download rate cap func NewStaticLimiter(l Limits) Limiter { var ( - upstreamBucket *ratelimit.Bucket - downstreamBucket *ratelimit.Bucket + upstreamBucket *rate.Limiter + downstreamBucket *rate.Limiter ) if l.UploadKb > 0 { - upstreamBucket = ratelimit.NewBucketWithRate(toByteRate(l.UploadKb), int64(toByteRate(l.UploadKb))) + upstreamBucket = rate.NewLimiter(rate.Limit(toByteRate(l.UploadKb)), int(toByteRate(l.UploadKb))) } if l.DownloadKb > 0 { - downstreamBucket = ratelimit.NewBucketWithRate(toByteRate(l.DownloadKb), int64(toByteRate(l.DownloadKb))) + downstreamBucket = rate.NewLimiter(rate.Limit(toByteRate(l.DownloadKb)), int(toByteRate(l.DownloadKb))) } return staticLimiter{ @@ -95,18 +96,55 @@ func (l staticLimiter) Transport(rt http.RoundTripper) http.RoundTripper { }) } -func (l staticLimiter) limitReader(r io.Reader, b *ratelimit.Bucket) io.Reader { +func (l staticLimiter) limitReader(r io.Reader, b *rate.Limiter) io.Reader { if b == nil { return r } - return ratelimit.Reader(r, b) + return &rateLimitedReader{r, b} } -func (l staticLimiter) limitWriter(w io.Writer, b *ratelimit.Bucket) io.Writer { +type rateLimitedReader struct { + reader io.Reader + bucket *rate.Limiter +} + +func (r *rateLimitedReader) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + if err := consumeTokens(n, r.bucket); err != nil { + return n, err + } + return n, err +} + +func (l staticLimiter) limitWriter(w io.Writer, b *rate.Limiter) io.Writer { if b == nil { return w } - return ratelimit.Writer(w, b) + return &rateLimitedWriter{w, b} +} + +type rateLimitedWriter struct { + writer io.Writer + bucket *rate.Limiter +} + +func (w *rateLimitedWriter) Write(buf []byte) (int, error) { + if err := consumeTokens(len(buf), w.bucket); err != nil { + return 0, err + } + return w.writer.Write(buf) +} + +func consumeTokens(tokens int, bucket *rate.Limiter) error { + // bucket allows waiting for at most Burst() tokens at once + maxWait := bucket.Burst() + for tokens > maxWait { + if err := bucket.WaitN(context.Background(), maxWait); err != nil { + return err + } + tokens -= maxWait + } + return bucket.WaitN(context.Background(), tokens) } func toByteRate(val int) float64 { diff --git a/internal/backend/limiter/static_limiter_test.go b/internal/backend/limiter/static_limiter_test.go index 564b6a00a..8a839518f 100644 --- a/internal/backend/limiter/static_limiter_test.go +++ b/internal/backend/limiter/static_limiter_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/restic/restic/internal/test" + "golang.org/x/time/rate" ) func TestLimiterWrapping(t *testing.T) { @@ -33,6 +34,38 @@ func TestLimiterWrapping(t *testing.T) { } } +func TestReadLimiter(t *testing.T) { + reader := bytes.NewReader(make([]byte, 300)) + limiter := rate.NewLimiter(rate.Limit(10000), int(100)) + limReader := rateLimitedReader{reader, limiter} + + n, err := limReader.Read([]byte{}) + test.OK(t, err) + test.Equals(t, n, 0) + + n, err = limReader.Read(make([]byte, 300)) + test.OK(t, err) + test.Equals(t, n, 300) + + n, err = limReader.Read([]byte{}) + test.Equals(t, err, io.EOF) + test.Equals(t, n, 0) +} + +func TestWriteLimiter(t *testing.T) { + writer := &bytes.Buffer{} + limiter := rate.NewLimiter(rate.Limit(10000), int(100)) + limReader := rateLimitedWriter{writer, limiter} + + n, err := limReader.Write([]byte{}) + test.OK(t, err) + test.Equals(t, n, 0) + + n, err = limReader.Write(make([]byte, 300)) + test.OK(t, err) + test.Equals(t, n, 300) +} + type tracedReadCloser struct { io.Reader Closed bool diff --git a/internal/backend/local/layout_test.go b/internal/backend/local/layout_test.go index a4fccd2cb..46f3996bb 100644 --- a/internal/backend/local/layout_test.go +++ b/internal/backend/local/layout_test.go @@ -5,7 +5,7 @@ import ( "path/filepath" "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" rtest "github.com/restic/restic/internal/test" ) @@ -49,7 +49,7 @@ func TestLayout(t *testing.T) { } packs := make(map[string]bool) - err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { packs[fi.Name] = false return nil }) diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index 4198102c2..b89f2ff44 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -12,10 +12,10 @@ import ( "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/limiter" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" - "github.com/restic/restic/internal/restic" "github.com/cenkalti/backoff/v4" ) @@ -24,11 +24,11 @@ import ( type Local struct { Config layout.Layout - backend.Modes + util.Modes } -// ensure statically that *Local implements restic.Backend. -var _ restic.Backend = &Local{} +// ensure statically that *Local implements backend.Backend. +var _ backend.Backend = &Local{} func NewFactory() location.Factory { return location.NewLimitedBackendFactory("local", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open)) @@ -42,8 +42,8 @@ func open(ctx context.Context, cfg Config) (*Local, error) { return nil, err } - fi, err := fs.Stat(l.Filename(restic.Handle{Type: restic.ConfigFile})) - m := backend.DeriveModesFromFileInfo(fi, err) + fi, err := fs.Stat(l.Filename(backend.Handle{Type: backend.ConfigFile})) + m := util.DeriveModesFromFileInfo(fi, err) debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir) return &Local{ @@ -70,7 +70,7 @@ func Create(ctx context.Context, cfg Config) (*Local, error) { } // test if config file already exists - _, err = fs.Lstat(be.Filename(restic.Handle{Type: restic.ConfigFile})) + _, err = fs.Lstat(be.Filename(backend.Handle{Type: backend.ConfigFile})) if err == nil { return nil, errors.New("config file already exists") } @@ -111,7 +111,7 @@ func (b *Local) IsNotExist(err error) bool { } // Save stores data in the backend at the handle. -func (b *Local) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) (err error) { +func (b *Local) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) (err error) { finalname := b.Filename(h) dir := filepath.Dir(finalname) @@ -194,7 +194,7 @@ func (b *Local) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) } } - // try to mark file as read-only to avoid accidential modifications + // try to mark file as read-only to avoid accidental modifications // ignore if the operation fails as some filesystems don't allow the chmod call // e.g. exfat and network file systems with certain mount options err = setFileReadonly(finalname, b.Modes.File) @@ -209,11 +209,11 @@ var tempFile = os.CreateTemp // Overridden by test. // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (b *Local) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - return backend.DefaultLoad(ctx, h, length, offset, b.openReader, fn) +func (b *Local) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + return util.DefaultLoad(ctx, h, length, offset, b.openReader, fn) } -func (b *Local) openReader(_ context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (b *Local) openReader(_ context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { f, err := fs.Open(b.Filename(h)) if err != nil { return nil, err @@ -235,17 +235,17 @@ func (b *Local) openReader(_ context.Context, h restic.Handle, length int, offse } // Stat returns information about a blob. -func (b *Local) Stat(_ context.Context, h restic.Handle) (restic.FileInfo, error) { +func (b *Local) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) { fi, err := fs.Stat(b.Filename(h)) if err != nil { - return restic.FileInfo{}, errors.WithStack(err) + return backend.FileInfo{}, errors.WithStack(err) } - return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil + return backend.FileInfo{Size: fi.Size(), Name: h.Name}, nil } // Remove removes the blob with the given name and type. -func (b *Local) Remove(_ context.Context, h restic.Handle) error { +func (b *Local) Remove(_ context.Context, h backend.Handle) error { fn := b.Filename(h) // reset read-only flag @@ -259,7 +259,7 @@ func (b *Local) Remove(_ context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) (err error) { +func (b *Local) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) (err error) { basedir, subdirs := b.Basedir(t) if subdirs { err = visitDirs(ctx, basedir, fn) @@ -279,7 +279,7 @@ func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.File // two levels of directory structure (including dir itself as the first level). // Also, visitDirs assumes it sees a directory full of directories, while // visitFiles wants a directory full or regular files. -func visitDirs(ctx context.Context, dir string, fn func(restic.FileInfo) error) error { +func visitDirs(ctx context.Context, dir string, fn func(backend.FileInfo) error) error { d, err := fs.Open(dir) if err != nil { return err @@ -306,7 +306,7 @@ func visitDirs(ctx context.Context, dir string, fn func(restic.FileInfo) error) return ctx.Err() } -func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error, ignoreNotADirectory bool) error { +func visitFiles(ctx context.Context, dir string, fn func(backend.FileInfo) error, ignoreNotADirectory bool) error { d, err := fs.Open(dir) if err != nil { return err @@ -340,7 +340,7 @@ func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error, default: } - err := fn(restic.FileInfo{ + err := fn(backend.FileInfo{ Name: fi.Name(), Size: fi.Size(), }) diff --git a/internal/backend/local/local_internal_test.go b/internal/backend/local/local_internal_test.go index 1e80e72ed..6cad26d0a 100644 --- a/internal/backend/local/local_internal_test.go +++ b/internal/backend/local/local_internal_test.go @@ -8,7 +8,7 @@ import ( "syscall" "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" rtest "github.com/restic/restic/internal/test" "github.com/cenkalti/backoff/v4" @@ -32,7 +32,7 @@ func TestNoSpacePermanent(t *testing.T) { rtest.OK(t, be.Close()) }() - h := restic.Handle{Type: restic.ConfigFile} + h := backend.Handle{Type: backend.ConfigFile} err = be.Save(context.Background(), h, nil) _, ok := err.(*backoff.PermanentError) rtest.Assert(t, ok, diff --git a/internal/backend/location/display_location_test.go b/internal/backend/location/display_location_test.go index 19502d85b..4011abbf0 100644 --- a/internal/backend/location/display_location_test.go +++ b/internal/backend/location/display_location_test.go @@ -3,15 +3,15 @@ package location_test import ( "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/location" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) func TestStripPassword(t *testing.T) { registry := location.NewRegistry() registry.Register( - location.NewHTTPBackendFactory[any, restic.Backend]("test", nil, + location.NewHTTPBackendFactory[any, backend.Backend]("test", nil, func(s string) string { return "cleaned" }, nil, nil, diff --git a/internal/backend/location/location_test.go b/internal/backend/location/location_test.go index b2623032e..fe550a586 100644 --- a/internal/backend/location/location_test.go +++ b/internal/backend/location/location_test.go @@ -3,8 +3,8 @@ package location_test import ( "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/location" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) @@ -13,7 +13,7 @@ type testConfig struct { } func testFactory() location.Factory { - return location.NewHTTPBackendFactory[testConfig, restic.Backend]( + return location.NewHTTPBackendFactory[testConfig, backend.Backend]( "local", func(s string) (*testConfig, error) { return &testConfig{loc: s}, nil diff --git a/internal/backend/location/registry.go b/internal/backend/location/registry.go index a8818bd73..b50371add 100644 --- a/internal/backend/location/registry.go +++ b/internal/backend/location/registry.go @@ -4,8 +4,8 @@ import ( "context" "net/http" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/limiter" - "github.com/restic/restic/internal/restic" ) type Registry struct { @@ -33,11 +33,11 @@ type Factory interface { Scheme() string ParseConfig(s string) (interface{}, error) StripPassword(s string) string - Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) - Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) + Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error) + Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error) } -type genericBackendFactory[C any, T restic.Backend] struct { +type genericBackendFactory[C any, T backend.Backend] struct { scheme string parseConfigFn func(s string) (*C, error) stripPasswordFn func(s string) string @@ -58,14 +58,14 @@ func (f *genericBackendFactory[C, T]) StripPassword(s string) string { } return s } -func (f *genericBackendFactory[C, T]) Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) { +func (f *genericBackendFactory[C, T]) Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error) { return f.createFn(ctx, *cfg.(*C), rt, lim) } -func (f *genericBackendFactory[C, T]) Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) { +func (f *genericBackendFactory[C, T]) Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error) { return f.openFn(ctx, *cfg.(*C), rt, lim) } -func NewHTTPBackendFactory[C any, T restic.Backend]( +func NewHTTPBackendFactory[C any, T backend.Backend]( scheme string, parseConfigFn func(s string) (*C, error), stripPasswordFn func(s string) string, @@ -85,7 +85,7 @@ func NewHTTPBackendFactory[C any, T restic.Backend]( } } -func NewLimitedBackendFactory[C any, T restic.Backend]( +func NewLimitedBackendFactory[C any, T backend.Backend]( scheme string, parseConfigFn func(s string) (*C, error), stripPasswordFn func(s string) string, diff --git a/internal/backend/logger/log.go b/internal/backend/logger/log.go index 6c860cfae..6fdf92295 100644 --- a/internal/backend/logger/log.go +++ b/internal/backend/logger/log.go @@ -4,18 +4,18 @@ import ( "context" "io" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/restic" ) type Backend struct { - restic.Backend + backend.Backend } -// statically ensure that Backend implements restic.Backend. -var _ restic.Backend = &Backend{} +// statically ensure that Backend implements backend.Backend. +var _ backend.Backend = &Backend{} -func New(be restic.Backend) *Backend { +func New(be backend.Backend) *Backend { return &Backend{Backend: be} } @@ -26,7 +26,7 @@ func (be *Backend) IsNotExist(err error) bool { } // Save adds new Data to the backend. -func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { debug.Log("Save(%v, %v)", h, rd.Length()) err := be.Backend.Save(ctx, h, rd) debug.Log(" save err %v", err) @@ -34,28 +34,28 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe } // Remove deletes a file from the backend. -func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (be *Backend) Remove(ctx context.Context, h backend.Handle) error { debug.Log("Remove(%v)", h) err := be.Backend.Remove(ctx, h) debug.Log(" remove err %v", err) return err } -func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(io.Reader) error) error { +func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(io.Reader) error) error { debug.Log("Load(%v, length %v, offset %v)", h, length, offset) err := be.Backend.Load(ctx, h, length, offset, fn) debug.Log(" load err %v", err) return err } -func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { debug.Log("Stat(%v)", h) fi, err := be.Backend.Stat(ctx, h) debug.Log(" stat err %v", err) return fi, err } -func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { debug.Log("List(%v)", t) err := be.Backend.List(ctx, t, fn) debug.Log(" list err %v", err) @@ -76,4 +76,4 @@ func (be *Backend) Close() error { return err } -func (be *Backend) Unwrap() restic.Backend { return be.Backend } +func (be *Backend) Unwrap() backend.Backend { return be.Backend } diff --git a/internal/backend/mem/mem_backend.go b/internal/backend/mem/mem_backend.go index 86ec48756..2698a8275 100644 --- a/internal/backend/mem/mem_backend.go +++ b/internal/backend/mem/mem_backend.go @@ -12,15 +12,15 @@ import ( "github.com/cespare/xxhash/v2" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" ) -type memMap map[restic.Handle][]byte +type memMap map[backend.Handle][]byte // make sure that MemoryBackend implements backend.Backend -var _ restic.Backend = &MemoryBackend{} +var _ backend.Backend = &MemoryBackend{} // NewFactory creates a persistent mem backend func NewFactory() location.Factory { @@ -69,12 +69,12 @@ func (be *MemoryBackend) IsNotExist(err error) bool { } // Save adds new Data to the backend. -func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *MemoryBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { be.m.Lock() defer be.m.Unlock() - h.ContainedBlobType = restic.InvalidBlob - if h.Type == restic.ConfigFile { + h.IsMetadata = false + if h.Type == backend.ConfigFile { h.Name = "" } @@ -112,16 +112,16 @@ func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *MemoryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn) +func (be *MemoryBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } -func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *MemoryBackend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { be.m.Lock() defer be.m.Unlock() - h.ContainedBlobType = restic.InvalidBlob - if h.Type == restic.ConfigFile { + h.IsMetadata = false + if h.Type == backend.ConfigFile { h.Name = "" } @@ -143,29 +143,29 @@ func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length } // Stat returns information about a file in the backend. -func (be *MemoryBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (be *MemoryBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { be.m.Lock() defer be.m.Unlock() - h.ContainedBlobType = restic.InvalidBlob - if h.Type == restic.ConfigFile { + h.IsMetadata = false + if h.Type == backend.ConfigFile { h.Name = "" } e, ok := be.data[h] if !ok { - return restic.FileInfo{}, errNotFound + return backend.FileInfo{}, errNotFound } - return restic.FileInfo{Size: int64(len(e)), Name: h.Name}, ctx.Err() + return backend.FileInfo{Size: int64(len(e)), Name: h.Name}, ctx.Err() } // Remove deletes a file from the backend. -func (be *MemoryBackend) Remove(ctx context.Context, h restic.Handle) error { +func (be *MemoryBackend) Remove(ctx context.Context, h backend.Handle) error { be.m.Lock() defer be.m.Unlock() - h.ContainedBlobType = restic.InvalidBlob + h.IsMetadata = false if _, ok := be.data[h]; !ok { return errNotFound } @@ -176,7 +176,7 @@ func (be *MemoryBackend) Remove(ctx context.Context, h restic.Handle) error { } // List returns a channel which yields entries from the backend. -func (be *MemoryBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *MemoryBackend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { entries := make(map[string]int64) be.m.Lock() @@ -190,7 +190,7 @@ func (be *MemoryBackend) List(ctx context.Context, t restic.FileType, fn func(re be.m.Unlock() for name, size := range entries { - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: name, Size: size, } diff --git a/internal/backend/mock/backend.go b/internal/backend/mock/backend.go index 875e55e71..57b1ede19 100644 --- a/internal/backend/mock/backend.go +++ b/internal/backend/mock/backend.go @@ -5,19 +5,19 @@ import ( "hash" "io" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" ) // Backend implements a mock backend. type Backend struct { CloseFn func() error IsNotExistFn func(err error) bool - SaveFn func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error - OpenReaderFn func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) - StatFn func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) - ListFn func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error - RemoveFn func(ctx context.Context, h restic.Handle) error + SaveFn func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error + OpenReaderFn func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) + StatFn func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) + ListFn func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error + RemoveFn func(ctx context.Context, h backend.Handle) error DeleteFn func(ctx context.Context) error ConnectionsFn func() uint LocationFn func() string @@ -84,7 +84,7 @@ func (m *Backend) IsNotExist(err error) bool { } // Save data in the backend. -func (m *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (m *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { if m.SaveFn == nil { return errors.New("not implemented") } @@ -94,7 +94,7 @@ func (m *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (m *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (m *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { rd, err := m.openReader(ctx, h, length, offset) if err != nil { return err @@ -107,7 +107,7 @@ func (m *Backend) Load(ctx context.Context, h restic.Handle, length int, offset return rd.Close() } -func (m *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (m *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { if m.OpenReaderFn == nil { return nil, errors.New("not implemented") } @@ -116,16 +116,16 @@ func (m *Backend) openReader(ctx context.Context, h restic.Handle, length int, o } // Stat an object in the backend. -func (m *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (m *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { if m.StatFn == nil { - return restic.FileInfo{}, errors.New("not implemented") + return backend.FileInfo{}, errors.New("not implemented") } return m.StatFn(ctx, h) } // List items of type t. -func (m *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (m *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { if m.ListFn == nil { return nil } @@ -134,7 +134,7 @@ func (m *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi } // Remove data from the backend. -func (m *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (m *Backend) Remove(ctx context.Context, h backend.Handle) error { if m.RemoveFn == nil { return errors.New("not implemented") } @@ -152,4 +152,4 @@ func (m *Backend) Delete(ctx context.Context) error { } // Make sure that Backend implements the backend interface. -var _ restic.Backend = &Backend{} +var _ backend.Backend = &Backend{} diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go index fd6f5b262..a41a89898 100644 --- a/internal/backend/rclone/backend.go +++ b/internal/backend/rclone/backend.go @@ -21,6 +21,7 @@ import ( "github.com/restic/restic/internal/backend/limiter" "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/rest" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "golang.org/x/net/http2" @@ -81,7 +82,7 @@ func run(command string, args ...string) (*StdioConn, *sync.WaitGroup, chan stru cmd.Stdin = r cmd.Stdout = w - bg, err := backend.StartForeground(cmd) + bg, err := util.StartForeground(cmd) // close rclone side of pipes errR := r.Close() errW := w.Close() @@ -93,7 +94,7 @@ func run(command string, args ...string) (*StdioConn, *sync.WaitGroup, chan stru err = errW } if err != nil { - if backend.IsErrDot(err) { + if util.IsErrDot(err) { return nil, nil, nil, nil, errors.Errorf("cannot implicitly run relative executable %v found in current directory, use -o rclone.program=./ to override", cmd.Path) } return nil, nil, nil, nil, err diff --git a/internal/backend/rclone/internal_test.go b/internal/backend/rclone/internal_test.go index 32fe850a0..34d52885e 100644 --- a/internal/backend/rclone/internal_test.go +++ b/internal/backend/rclone/internal_test.go @@ -5,8 +5,8 @@ import ( "os/exec" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -32,9 +32,9 @@ func TestRcloneExit(t *testing.T) { t.Log("killed rclone") for i := 0; i < 10; i++ { - _, err = be.Stat(context.TODO(), restic.Handle{ + _, err = be.Stat(context.TODO(), backend.Handle{ Name: "foo", - Type: restic.PackFile, + Type: backend.PackFile, }) rtest.Assert(t, err != nil, "expected an error") } diff --git a/internal/backend/readerat.go b/internal/backend/readerat.go index ff2e40393..f4164cc6e 100644 --- a/internal/backend/readerat.go +++ b/internal/backend/readerat.go @@ -6,13 +6,12 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" ) type backendReaderAt struct { ctx context.Context - be restic.Backend - h restic.Handle + be Backend + h Handle } func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) { @@ -22,12 +21,12 @@ func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) { // ReaderAt returns an io.ReaderAt for a file in the backend. The returned reader // should not escape the caller function to avoid unexpected interactions with the // embedded context -func ReaderAt(ctx context.Context, be restic.Backend, h restic.Handle) io.ReaderAt { +func ReaderAt(ctx context.Context, be Backend, h Handle) io.ReaderAt { return backendReaderAt{ctx: ctx, be: be, h: h} } // ReadAt reads from the backend handle h at the given position. -func ReadAt(ctx context.Context, be restic.Backend, h restic.Handle, offset int64, p []byte) (n int, err error) { +func ReadAt(ctx context.Context, be Backend, h Handle, offset int64, p []byte) (n int, err error) { debug.Log("ReadAt(%v) at %v, len %v", h, offset, len(p)) err = be.Load(ctx, h, len(p), offset, func(rd io.Reader) (ierr error) { diff --git a/internal/backend/rest/config.go b/internal/backend/rest/config.go index ba42a0220..8f17d444a 100644 --- a/internal/backend/rest/config.go +++ b/internal/backend/rest/config.go @@ -2,8 +2,10 @@ package rest import ( "net/url" + "os" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" ) @@ -70,3 +72,19 @@ func prepareURL(s string) string { } return s } + +var _ backend.ApplyEnvironmenter = &Config{} + +// ApplyEnvironment saves values from the environment to the config. +func (cfg *Config) ApplyEnvironment(prefix string) { + username := cfg.URL.User.Username() + _, pwdSet := cfg.URL.User.Password() + + // Only apply env variable values if neither username nor password are provided. + if username == "" && !pwdSet { + envName := os.Getenv(prefix + "RESTIC_REST_USERNAME") + envPwd := os.Getenv(prefix + "RESTIC_REST_PASSWORD") + + cfg.URL.User = url.UserPassword(envName, envPwd) + } +} diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index 8391df681..1d1769b56 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -14,13 +14,13 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" ) -// make sure the rest backend implements restic.Backend -var _ restic.Backend = &Backend{} +// make sure the rest backend implements backend.Backend +var _ backend.Backend = &Backend{} // Backend uses the REST protocol to access data stored on a server. type Backend struct { @@ -65,7 +65,7 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, er return nil, err } - _, err = be.Stat(ctx, restic.Handle{Type: restic.ConfigFile}) + _, err = be.Stat(ctx, backend.Handle{Type: backend.ConfigFile}) if err == nil { return nil, errors.New("config file already exists") } @@ -118,7 +118,7 @@ func (b *Backend) HasAtomicReplace() bool { } // Save stores data in the backend at the handle. -func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -147,7 +147,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea return errors.WithStack(err) } - if resp.StatusCode != 200 { + if resp.StatusCode != http.StatusOK { return errors.Errorf("server response unexpected: %v (%v)", resp.Status, resp.StatusCode) } @@ -157,7 +157,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea // notExistError is returned whenever the requested file does not exist on the // server. type notExistError struct { - restic.Handle + backend.Handle } func (e *notExistError) Error() string { @@ -172,7 +172,7 @@ func (b *Backend) IsNotExist(err error) bool { // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { r, err := b.openReader(ctx, h, length, offset) if err != nil { return err @@ -201,7 +201,7 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset return err } -func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (b *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { req, err := http.NewRequestWithContext(ctx, "GET", b.Filename(h), nil) if err != nil { return nil, errors.WithStack(err) @@ -229,7 +229,7 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o return nil, ¬ExistError{h} } - if resp.StatusCode != 200 && resp.StatusCode != 206 { + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { _ = resp.Body.Close() return nil, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status) } @@ -238,37 +238,37 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o } // Stat returns information about a blob. -func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { req, err := http.NewRequestWithContext(ctx, http.MethodHead, b.Filename(h), nil) if err != nil { - return restic.FileInfo{}, errors.WithStack(err) + return backend.FileInfo{}, errors.WithStack(err) } req.Header.Set("Accept", ContentTypeV2) resp, err := b.client.Do(req) if err != nil { - return restic.FileInfo{}, errors.WithStack(err) + return backend.FileInfo{}, errors.WithStack(err) } _, _ = io.Copy(io.Discard, resp.Body) if err = resp.Body.Close(); err != nil { - return restic.FileInfo{}, errors.Wrap(err, "Close") + return backend.FileInfo{}, errors.Wrap(err, "Close") } if resp.StatusCode == http.StatusNotFound { _ = resp.Body.Close() - return restic.FileInfo{}, ¬ExistError{h} + return backend.FileInfo{}, ¬ExistError{h} } - if resp.StatusCode != 200 { - return restic.FileInfo{}, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status) + if resp.StatusCode != http.StatusOK { + return backend.FileInfo{}, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status) } if resp.ContentLength < 0 { - return restic.FileInfo{}, errors.New("negative content length") + return backend.FileInfo{}, errors.New("negative content length") } - bi := restic.FileInfo{ + bi := backend.FileInfo{ Size: resp.ContentLength, Name: h.Name, } @@ -277,7 +277,7 @@ func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, e } // Remove removes the blob with the given name and type. -func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (b *Backend) Remove(ctx context.Context, h backend.Handle) error { req, err := http.NewRequestWithContext(ctx, "DELETE", b.Filename(h), nil) if err != nil { return errors.WithStack(err) @@ -295,7 +295,7 @@ func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { return ¬ExistError{h} } - if resp.StatusCode != 200 { + if resp.StatusCode != http.StatusOK { return errors.Errorf("blob not removed, server response: %v (%v)", resp.Status, resp.StatusCode) } @@ -309,8 +309,8 @@ func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - url := b.Dirname(restic.Handle{Type: t}) +func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { + url := b.Dirname(backend.Handle{Type: t}) if !strings.HasSuffix(url, "/") { url += "/" } @@ -327,7 +327,12 @@ func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi return errors.Wrap(err, "List") } - if resp.StatusCode != 200 { + if resp.StatusCode == http.StatusNotFound { + // ignore missing directories + return nil + } + + if resp.StatusCode != http.StatusOK { return errors.Errorf("List failed, server response: %v (%v)", resp.Status, resp.StatusCode) } @@ -341,7 +346,7 @@ func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi // listv1 uses the REST protocol v1, where a list HTTP request (e.g. `GET // /data/`) only returns the names of the files, so we need to issue an HTTP // HEAD request for each file. -func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error { +func (b *Backend) listv1(ctx context.Context, t backend.FileType, resp *http.Response, fn func(backend.FileInfo) error) error { debug.Log("parsing API v1 response") dec := json.NewDecoder(resp.Body) var list []string @@ -350,7 +355,7 @@ func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Resp } for _, m := range list { - fi, err := b.Stat(ctx, restic.Handle{Name: m, Type: t}) + fi, err := b.Stat(ctx, backend.Handle{Name: m, Type: t}) if err != nil { return err } @@ -375,7 +380,7 @@ func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Resp // listv2 uses the REST protocol v2, where a list HTTP request (e.g. `GET // /data/`) returns the names and sizes of all files. -func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(restic.FileInfo) error) error { +func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(backend.FileInfo) error) error { debug.Log("parsing API v2 response") dec := json.NewDecoder(resp.Body) @@ -392,7 +397,7 @@ func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(resti return ctx.Err() } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: item.Name, Size: item.Size, } @@ -419,5 +424,5 @@ func (b *Backend) Close() error { // Delete removes all data in the backend. func (b *Backend) Delete(ctx context.Context) error { - return backend.DefaultDelete(ctx, b) + return util.DefaultDelete(ctx, b) } diff --git a/internal/backend/rest/rest_int_test.go b/internal/backend/rest/rest_int_test.go index e7810c5e3..853a852c7 100644 --- a/internal/backend/rest/rest_int_test.go +++ b/internal/backend/rest/rest_int_test.go @@ -10,8 +10,8 @@ import ( "strconv" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/rest" - "github.com/restic/restic/internal/restic" ) func TestListAPI(t *testing.T) { @@ -22,7 +22,7 @@ func TestListAPI(t *testing.T) { Data string // response data Requests int - Result []restic.FileInfo + Result []backend.FileInfo }{ { Name: "content-type-unknown", @@ -32,7 +32,7 @@ func TestListAPI(t *testing.T) { "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b" ]`, - Result: []restic.FileInfo{ + Result: []backend.FileInfo{ {Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386}, {Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214}, {Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393}, @@ -47,7 +47,7 @@ func TestListAPI(t *testing.T) { "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b" ]`, - Result: []restic.FileInfo{ + Result: []backend.FileInfo{ {Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386}, {Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214}, {Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393}, @@ -62,7 +62,7 @@ func TestListAPI(t *testing.T) { {"name": "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", "size": 1002}, {"name": "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", "size": 1003} ]`, - Result: []restic.FileInfo{ + Result: []backend.FileInfo{ {Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 1001}, {Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 1002}, {Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 1003}, @@ -122,8 +122,8 @@ func TestListAPI(t *testing.T) { t.Fatal(err) } - var list []restic.FileInfo - err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + var list []backend.FileInfo + err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { list = append(list, fi) return nil }) diff --git a/internal/backend/retry/backend_retry.go b/internal/backend/retry/backend_retry.go index 9c51efedc..c63338fb6 100644 --- a/internal/backend/retry/backend_retry.go +++ b/internal/backend/retry/backend_retry.go @@ -7,27 +7,27 @@ import ( "time" "github.com/cenkalti/backoff/v4" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/restic" ) // Backend retries operations on the backend in case of an error with a // backoff. type Backend struct { - restic.Backend + backend.Backend MaxTries int Report func(string, error, time.Duration) Success func(string, int) } -// statically ensure that RetryBackend implements restic.Backend. -var _ restic.Backend = &Backend{} +// statically ensure that RetryBackend implements backend.Backend. +var _ backend.Backend = &Backend{} // New wraps be with a backend that retries operations after a // backoff. report is called with a description and the error, if one occurred. // success is called with the number of retries before a successful operation // (it is not called if it succeeded on the first try) -func New(be restic.Backend, maxTries int, report func(string, error, time.Duration), success func(string, int)) *Backend { +func New(be backend.Backend, maxTries int, report func(string, error, time.Duration), success func(string, int)) *Backend { return &Backend{ Backend: be, MaxTries: maxTries, @@ -92,7 +92,7 @@ func (be *Backend) retry(ctx context.Context, msg string, f func() error) error } // Save stores the data in the backend under the given handle. -func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { return be.retry(ctx, fmt.Sprintf("Save(%v)", h), func() error { err := rd.Rewind() if err != nil { @@ -125,15 +125,19 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe // given offset. If length is larger than zero, only a portion of the file // is returned. rd must be closed after use. If an error is returned, the // ReadCloser must be nil. -func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) { +func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) { return be.retry(ctx, fmt.Sprintf("Load(%v, %v, %v)", h, length, offset), func() error { - return be.Backend.Load(ctx, h, length, offset, consumer) + err := be.Backend.Load(ctx, h, length, offset, consumer) + if be.Backend.IsNotExist(err) { + return backoff.Permanent(err) + } + return err }) } // Stat returns information about the File identified by h. -func (be *Backend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInfo, err error) { +func (be *Backend) Stat(ctx context.Context, h backend.Handle) (fi backend.FileInfo, err error) { err = be.retry(ctx, fmt.Sprintf("Stat(%v)", h), func() error { var innerError error @@ -149,7 +153,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInf } // Remove removes a File with type t and name. -func (be *Backend) Remove(ctx context.Context, h restic.Handle) (err error) { +func (be *Backend) Remove(ctx context.Context, h backend.Handle) (err error) { return be.retry(ctx, fmt.Sprintf("Remove(%v)", h), func() error { return be.Backend.Remove(ctx, h) }) @@ -159,7 +163,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) (err error) { // error is returned by the underlying backend, the request is retried. When fn // returns an error, the operation is aborted and the error is returned to the // caller. -func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { // create a new context that we can cancel when fn returns an error, so // that listing is aborted listCtx, cancel := context.WithCancel(ctx) @@ -169,7 +173,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F var innerErr error // remember when fn returned an error, so we can return that to the caller err := be.retry(listCtx, fmt.Sprintf("List(%v)", t), func() error { - return be.Backend.List(ctx, t, func(fi restic.FileInfo) error { + return be.Backend.List(ctx, t, func(fi backend.FileInfo) error { if _, ok := listed[fi.Name]; ok { return nil } @@ -192,6 +196,6 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F return err } -func (be *Backend) Unwrap() restic.Backend { +func (be *Backend) Unwrap() backend.Backend { return be.Backend } diff --git a/internal/backend/retry/backend_retry_test.go b/internal/backend/retry/backend_retry_test.go index 9f2f39589..405cdfa59 100644 --- a/internal/backend/retry/backend_retry_test.go +++ b/internal/backend/retry/backend_retry_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/cenkalti/backoff/v4" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mock" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -18,7 +19,7 @@ func TestBackendSaveRetry(t *testing.T) { buf := bytes.NewBuffer(nil) errcount := 0 be := &mock.Backend{ - SaveFn: func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + SaveFn: func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { if errcount == 0 { errcount++ _, err := io.CopyN(io.Discard, rd, 120) @@ -38,7 +39,7 @@ func TestBackendSaveRetry(t *testing.T) { retryBackend := New(be, 10, nil, nil) data := test.Random(23, 5*1024*1024+11241) - err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher())) + err := retryBackend.Save(context.TODO(), backend.Handle{}, backend.NewByteReader(data, be.Hasher())) if err != nil { t.Fatal(err) } @@ -56,14 +57,14 @@ func TestBackendSaveRetryAtomic(t *testing.T) { errcount := 0 calledRemove := false be := &mock.Backend{ - SaveFn: func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + SaveFn: func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { if errcount == 0 { errcount++ return errors.New("injected error") } return nil }, - RemoveFn: func(ctx context.Context, h restic.Handle) error { + RemoveFn: func(ctx context.Context, h backend.Handle) error { calledRemove = true return nil }, @@ -74,7 +75,7 @@ func TestBackendSaveRetryAtomic(t *testing.T) { retryBackend := New(be, 10, nil, nil) data := test.Random(23, 5*1024*1024+11241) - err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher())) + err := retryBackend.Save(context.TODO(), backend.Handle{}, backend.NewByteReader(data, be.Hasher())) if err != nil { t.Fatal(err) } @@ -91,15 +92,15 @@ func TestBackendListRetry(t *testing.T) { retry := 0 be := &mock.Backend{ - ListFn: func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { + ListFn: func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { // fail during first retry, succeed during second retry++ if retry == 1 { - _ = fn(restic.FileInfo{Name: ID1}) + _ = fn(backend.FileInfo{Name: ID1}) return errors.New("test list error") } - _ = fn(restic.FileInfo{Name: ID1}) - _ = fn(restic.FileInfo{Name: ID2}) + _ = fn(backend.FileInfo{Name: ID1}) + _ = fn(backend.FileInfo{Name: ID2}) return nil }, } @@ -108,7 +109,7 @@ func TestBackendListRetry(t *testing.T) { retryBackend := New(be, 10, nil, nil) var listed []string - err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { listed = append(listed, fi.Name) return nil }) @@ -121,10 +122,10 @@ func TestBackendListRetryErrorFn(t *testing.T) { var names = []string{"id1", "id2", "foo", "bar"} be := &mock.Backend{ - ListFn: func(ctx context.Context, tpe restic.FileType, fn func(restic.FileInfo) error) error { + ListFn: func(ctx context.Context, tpe backend.FileType, fn func(backend.FileInfo) error) error { t.Logf("List called for %v", tpe) for _, name := range names { - err := fn(restic.FileInfo{Name: name}) + err := fn(backend.FileInfo{Name: name}) if err != nil { return err } @@ -141,7 +142,7 @@ func TestBackendListRetryErrorFn(t *testing.T) { var listed []string run := 0 - err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { t.Logf("fn called for %v", fi.Name) run++ // return an error for the third item in the list @@ -172,7 +173,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) { retries := 0 be := &mock.Backend{ - ListFn: func(ctx context.Context, tpe restic.FileType, fn func(restic.FileInfo) error) error { + ListFn: func(ctx context.Context, tpe backend.FileType, fn func(backend.FileInfo) error) error { t.Logf("List called for %v, retries %v", tpe, retries) retries++ for i, name := range names { @@ -180,7 +181,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) { return ErrBackendTest } - err := fn(restic.FileInfo{Name: name}) + err := fn(backend.FileInfo{Name: name}) if err != nil { return err } @@ -195,7 +196,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) { retryBackend := New(be, maxRetries, nil, nil) var listed []string - err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { t.Logf("fn called for %v", fi.Name) listed = append(listed, fi.Name) return nil @@ -252,7 +253,7 @@ func TestBackendLoadRetry(t *testing.T) { attempt := 0 be := mock.NewBackend() - be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { // returns failing reader on first invocation, good reader on subsequent invocations attempt++ if attempt > 1 { @@ -265,7 +266,7 @@ func TestBackendLoadRetry(t *testing.T) { retryBackend := New(be, 10, nil, nil) var buf []byte - err := retryBackend.Load(context.TODO(), restic.Handle{}, 0, 0, func(rd io.Reader) (err error) { + err := retryBackend.Load(context.TODO(), backend.Handle{}, 0, 0, func(rd io.Reader) (err error) { buf, err = io.ReadAll(rd) return err }) @@ -274,19 +275,19 @@ func TestBackendLoadRetry(t *testing.T) { test.Equals(t, 2, attempt) } -func TestBackendStatNotExists(t *testing.T) { - // stat should not retry if the error matches IsNotExist +func TestBackendLoadNotExists(t *testing.T) { + // load should not retry if the error matches IsNotExist notFound := errors.New("not found") attempt := 0 be := mock.NewBackend() - be.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { + be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { attempt++ if attempt > 1 { t.Fail() - return restic.FileInfo{}, errors.New("must not retry") + return nil, errors.New("must not retry") } - return restic.FileInfo{}, notFound + return nil, notFound } be.IsNotExistFn = func(err error) bool { return errors.Is(err, notFound) @@ -295,7 +296,35 @@ func TestBackendStatNotExists(t *testing.T) { TestFastRetries(t) retryBackend := New(be, 10, nil, nil) - _, err := retryBackend.Stat(context.TODO(), restic.Handle{}) + err := retryBackend.Load(context.TODO(), backend.Handle{}, 0, 0, func(rd io.Reader) (err error) { + return nil + }) + test.Assert(t, be.IsNotExistFn(err), "unexpected error %v", err) + test.Equals(t, 1, attempt) +} + +func TestBackendStatNotExists(t *testing.T) { + // stat should not retry if the error matches IsNotExist + notFound := errors.New("not found") + attempt := 0 + + be := mock.NewBackend() + be.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { + attempt++ + if attempt > 1 { + t.Fail() + return backend.FileInfo{}, errors.New("must not retry") + } + return backend.FileInfo{}, notFound + } + be.IsNotExistFn = func(err error) bool { + return errors.Is(err, notFound) + } + + TestFastRetries(t) + retryBackend := New(be, 10, nil, nil) + + _, err := retryBackend.Stat(context.TODO(), backend.Handle{}) test.Assert(t, be.IsNotExistFn(err), "unexpected error %v", err) test.Equals(t, 1, attempt) } @@ -309,7 +338,7 @@ func TestBackendCanceledContext(t *testing.T) { // check that we received the expected context canceled error instead TestFastRetries(t) retryBackend := New(mock.NewBackend(), 2, nil, nil) - h := restic.Handle{Type: restic.PackFile, Name: restic.NewRandomID().String()} + h := backend.Handle{Type: backend.PackFile, Name: restic.NewRandomID().String()} // create an already canceled context ctx, cancel := context.WithCancel(context.Background()) @@ -318,15 +347,15 @@ func TestBackendCanceledContext(t *testing.T) { _, err := retryBackend.Stat(ctx, h) assertIsCanceled(t, err) - err = retryBackend.Save(ctx, h, restic.NewByteReader([]byte{}, nil)) + err = retryBackend.Save(ctx, h, backend.NewByteReader([]byte{}, nil)) assertIsCanceled(t, err) err = retryBackend.Remove(ctx, h) assertIsCanceled(t, err) - err = retryBackend.Load(ctx, restic.Handle{}, 0, 0, func(rd io.Reader) (err error) { + err = retryBackend.Load(ctx, backend.Handle{}, 0, 0, func(rd io.Reader) (err error) { return nil }) assertIsCanceled(t, err) - err = retryBackend.List(ctx, restic.PackFile, func(restic.FileInfo) error { + err = retryBackend.List(ctx, backend.PackFile, func(backend.FileInfo) error { return nil }) assertIsCanceled(t, err) diff --git a/internal/restic/rewind_reader.go b/internal/backend/rewind_reader.go similarity index 99% rename from internal/restic/rewind_reader.go rename to internal/backend/rewind_reader.go index c27724e02..762b530aa 100644 --- a/internal/restic/rewind_reader.go +++ b/internal/backend/rewind_reader.go @@ -1,4 +1,4 @@ -package restic +package backend import ( "bytes" diff --git a/internal/restic/rewind_reader_test.go b/internal/backend/rewind_reader_test.go similarity index 99% rename from internal/restic/rewind_reader_test.go rename to internal/backend/rewind_reader_test.go index 8ec79ddcd..2ee287596 100644 --- a/internal/restic/rewind_reader_test.go +++ b/internal/backend/rewind_reader_test.go @@ -1,4 +1,4 @@ -package restic +package backend import ( "bytes" diff --git a/internal/backend/s3/config.go b/internal/backend/s3/config.go index 8dcad9eee..b4d44399f 100644 --- a/internal/backend/s3/config.go +++ b/internal/backend/s3/config.go @@ -6,9 +6,9 @@ import ( "path" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" ) // Config contains all configuration necessary to connect to an s3 compatible @@ -94,7 +94,7 @@ func createConfig(endpoint, bucket, prefix string, useHTTP bool) (*Config, error return &cfg, nil } -var _ restic.ApplyEnvironmenter = &Config{} +var _ backend.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. func (cfg *Config) ApplyEnvironment(prefix string) { diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index 3fe32d215..98879d0df 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -14,9 +14,9 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" @@ -30,7 +30,7 @@ type Backend struct { } // make sure that *Backend implements backend.Backend -var _ restic.Backend = &Backend{} +var _ backend.Backend = &Backend{} func NewFactory() location.Factory { return location.NewHTTPBackendFactory("s3", ParseConfig, location.NoPassword, Create, Open) @@ -127,13 +127,13 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro // Open opens the S3 backend at bucket and region. The bucket is created if it // does not exist yet. -func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { return open(ctx, cfg, rt) } // Create opens the S3 backend at bucket and region and creates the bucket if // it does not exist yet. -func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { be, err := open(ctx, cfg, rt) if err != nil { return nil, errors.Wrap(err, "open") @@ -272,7 +272,7 @@ func (be *Backend) Path() string { } // Save stores data in the backend at the handle. -func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { objName := be.Filename(h) opts := minio.PutObjectOptions{StorageClass: be.cfg.StorageClass} @@ -294,14 +294,14 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { ctx, cancel := context.WithCancel(ctx) defer cancel() - return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn) + return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } -func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { objName := be.Filename(h) opts := minio.GetObjectOptions{} @@ -326,7 +326,7 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, } // Stat returns information about a blob. -func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { +func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) { objName := be.Filename(h) var obj *minio.Object @@ -334,7 +334,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf obj, err = be.client.GetObject(ctx, be.cfg.Bucket, objName, opts) if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "client.GetObject") + return backend.FileInfo{}, errors.Wrap(err, "client.GetObject") } // make sure that the object is closed properly. @@ -347,14 +347,14 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf fi, err := obj.Stat() if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "Stat") + return backend.FileInfo{}, errors.Wrap(err, "Stat") } - return restic.FileInfo{Size: fi.Size, Name: h.Name}, nil + return backend.FileInfo{Size: fi.Size, Name: h.Name}, nil } // Remove removes the blob with the given name and type. -func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (be *Backend) Remove(ctx context.Context, h backend.Handle) error { objName := be.Filename(h) err := be.client.RemoveObject(ctx, be.cfg.Bucket, objName, minio.RemoveObjectOptions{}) @@ -368,7 +368,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { prefix, recursive := be.Basedir(t) // make sure prefix ends with a slash @@ -400,7 +400,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F continue } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: path.Base(m), Size: obj.Size, } @@ -424,14 +424,14 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F // Delete removes all restic keys in the bucket. It will not remove the bucket itself. func (be *Backend) Delete(ctx context.Context) error { - return backend.DefaultDelete(ctx, be) + return util.DefaultDelete(ctx, be) } // Close does nothing func (be *Backend) Close() error { return nil } // Rename moves a file based on the new layout l. -func (be *Backend) Rename(ctx context.Context, h restic.Handle, l layout.Layout) error { +func (be *Backend) Rename(ctx context.Context, h backend.Handle, l layout.Layout) error { debug.Log("Rename %v to %v", h, l) oldname := be.Filename(h) newname := l.Filename(h) diff --git a/internal/backend/s3/s3_test.go b/internal/backend/s3/s3_test.go index 3051d8ddb..470088e07 100644 --- a/internal/backend/s3/s3_test.go +++ b/internal/backend/s3/s3_test.go @@ -14,11 +14,11 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/backend/test" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -117,7 +117,7 @@ func newMinioTestSuite(t testing.TB) (*test.Suite[s3.Config], func()) { return &cfg, nil }, - Factory: location.NewHTTPBackendFactory("s3", s3.ParseConfig, location.NoPassword, func(ctx context.Context, cfg s3.Config, rt http.RoundTripper) (be restic.Backend, err error) { + Factory: location.NewHTTPBackendFactory("s3", s3.ParseConfig, location.NoPassword, func(ctx context.Context, cfg s3.Config, rt http.RoundTripper) (be backend.Backend, err error) { for i := 0; i < 10; i++ { be, err = s3.Create(ctx, cfg, rt) if err != nil { diff --git a/internal/backend/sema/backend.go b/internal/backend/sema/backend.go index dd4859ed1..1d69c52ac 100644 --- a/internal/backend/sema/backend.go +++ b/internal/backend/sema/backend.go @@ -3,23 +3,25 @@ package sema import ( "context" "io" + "sync" "github.com/cenkalti/backoff/v4" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" ) -// make sure that connectionLimitedBackend implements restic.Backend -var _ restic.Backend = &connectionLimitedBackend{} +// make sure that connectionLimitedBackend implements backend.Backend +var _ backend.Backend = &connectionLimitedBackend{} // connectionLimitedBackend limits the number of concurrent operations. type connectionLimitedBackend struct { - restic.Backend - sem semaphore + backend.Backend + sem semaphore + freezeLock sync.Mutex } // NewBackend creates a backend that limits the concurrent operations on the underlying backend -func NewBackend(be restic.Backend) restic.Backend { +func NewBackend(be backend.Backend) backend.Backend { sem, err := newSemaphore(be.Connections()) if err != nil { panic(err) @@ -33,29 +35,47 @@ func NewBackend(be restic.Backend) restic.Backend { // typeDependentLimit acquire a token unless the FileType is a lock file. The returned function // must be called to release the token. -func (be *connectionLimitedBackend) typeDependentLimit(t restic.FileType) func() { +func (be *connectionLimitedBackend) typeDependentLimit(t backend.FileType) func() { // allow concurrent lock file operations to ensure that the lock refresh is always possible - if t == restic.LockFile { + if t == backend.LockFile { return func() {} } be.sem.GetToken() + // prevent token usage while the backend is frozen + be.freezeLock.Lock() + defer be.freezeLock.Unlock() + return be.sem.ReleaseToken } +// Freeze blocks all backend operations except those on lock files +func (be *connectionLimitedBackend) Freeze() { + be.freezeLock.Lock() +} + +// Unfreeze allows all backend operations to continue +func (be *connectionLimitedBackend) Unfreeze() { + be.freezeLock.Unlock() +} + // Save adds new Data to the backend. -func (be *connectionLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *connectionLimitedBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { if err := h.Valid(); err != nil { return backoff.Permanent(err) } defer be.typeDependentLimit(h.Type)() + if ctx.Err() != nil { + return ctx.Err() + } + return be.Backend.Save(ctx, h, rd) } // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *connectionLimitedBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *connectionLimitedBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { if err := h.Valid(); err != nil { return backoff.Permanent(err) } @@ -68,31 +88,43 @@ func (be *connectionLimitedBackend) Load(ctx context.Context, h restic.Handle, l defer be.typeDependentLimit(h.Type)() + if ctx.Err() != nil { + return ctx.Err() + } + return be.Backend.Load(ctx, h, length, offset, fn) } // Stat returns information about a file in the backend. -func (be *connectionLimitedBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (be *connectionLimitedBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { if err := h.Valid(); err != nil { - return restic.FileInfo{}, backoff.Permanent(err) + return backend.FileInfo{}, backoff.Permanent(err) } defer be.typeDependentLimit(h.Type)() + if ctx.Err() != nil { + return backend.FileInfo{}, ctx.Err() + } + return be.Backend.Stat(ctx, h) } // Remove deletes a file from the backend. -func (be *connectionLimitedBackend) Remove(ctx context.Context, h restic.Handle) error { +func (be *connectionLimitedBackend) Remove(ctx context.Context, h backend.Handle) error { if err := h.Valid(); err != nil { return backoff.Permanent(err) } defer be.typeDependentLimit(h.Type)() + if ctx.Err() != nil { + return ctx.Err() + } + return be.Backend.Remove(ctx, h) } -func (be *connectionLimitedBackend) Unwrap() restic.Backend { +func (be *connectionLimitedBackend) Unwrap() backend.Backend { return be.Backend } diff --git a/internal/backend/sema/backend_test.go b/internal/backend/sema/backend_test.go index dc599b7f8..d220f48a3 100644 --- a/internal/backend/sema/backend_test.go +++ b/internal/backend/sema/backend_test.go @@ -3,41 +3,42 @@ package sema_test import ( "context" "io" + "sync" "sync/atomic" "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mock" "github.com/restic/restic/internal/backend/sema" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" "golang.org/x/sync/errgroup" ) func TestParameterValidationSave(t *testing.T) { m := mock.NewBackend() - m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { return nil } be := sema.NewBackend(m) - err := be.Save(context.TODO(), restic.Handle{}, nil) + err := be.Save(context.TODO(), backend.Handle{}, nil) test.Assert(t, err != nil, "Save() with invalid handle did not return an error") } func TestParameterValidationLoad(t *testing.T) { m := mock.NewBackend() - m.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + m.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { return io.NopCloser(nil), nil } be := sema.NewBackend(m) nilCb := func(rd io.Reader) error { return nil } - err := be.Load(context.TODO(), restic.Handle{}, 10, 0, nilCb) + err := be.Load(context.TODO(), backend.Handle{}, 10, 0, nilCb) test.Assert(t, err != nil, "Load() with invalid handle did not return an error") - h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + h := backend.Handle{Type: backend.PackFile, Name: "foobar"} err = be.Load(context.TODO(), h, 10, -1, nilCb) test.Assert(t, err != nil, "Save() with negative offset did not return an error") err = be.Load(context.TODO(), h, -1, 0, nilCb) @@ -46,23 +47,23 @@ func TestParameterValidationLoad(t *testing.T) { func TestParameterValidationStat(t *testing.T) { m := mock.NewBackend() - m.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { - return restic.FileInfo{}, nil + m.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { + return backend.FileInfo{}, nil } be := sema.NewBackend(m) - _, err := be.Stat(context.TODO(), restic.Handle{}) + _, err := be.Stat(context.TODO(), backend.Handle{}) test.Assert(t, err != nil, "Stat() with invalid handle did not return an error") } func TestParameterValidationRemove(t *testing.T) { m := mock.NewBackend() - m.RemoveFn = func(ctx context.Context, h restic.Handle) error { + m.RemoveFn = func(ctx context.Context, h backend.Handle) error { return nil } be := sema.NewBackend(m) - err := be.Remove(context.TODO(), restic.Handle{}) + err := be.Remove(context.TODO(), backend.Handle{}) test.Assert(t, err != nil, "Remove() with invalid handle did not return an error") } @@ -70,7 +71,7 @@ func TestUnwrap(t *testing.T) { m := mock.NewBackend() be := sema.NewBackend(m) - unwrapper := be.(restic.BackendUnwrapper) + unwrapper := be.(backend.Unwrapper) test.Assert(t, unwrapper.Unwrap() == m, "Unwrap() returned wrong backend") } @@ -99,7 +100,7 @@ func countingBlocker() (func(), func(int) int) { return wait, unblock } -func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(be restic.Backend) func() error, unblock func(int) int, isUnlimited bool) { +func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(be backend.Backend) func() error, unblock func(int) int, isUnlimited bool) { expectBlocked := int(2) workerCount := expectBlocked + 1 @@ -124,13 +125,13 @@ func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(b func TestConcurrencyLimitSave(t *testing.T) { wait, unblock := countingBlocker() concurrencyTester(t, func(m *mock.Backend) { - m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { wait() return nil } - }, func(be restic.Backend) func() error { + }, func(be backend.Backend) func() error { return func() error { - h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + h := backend.Handle{Type: backend.PackFile, Name: "foobar"} return be.Save(context.TODO(), h, nil) } }, unblock, false) @@ -139,13 +140,13 @@ func TestConcurrencyLimitSave(t *testing.T) { func TestConcurrencyLimitLoad(t *testing.T) { wait, unblock := countingBlocker() concurrencyTester(t, func(m *mock.Backend) { - m.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + m.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { wait() return io.NopCloser(nil), nil } - }, func(be restic.Backend) func() error { + }, func(be backend.Backend) func() error { return func() error { - h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + h := backend.Handle{Type: backend.PackFile, Name: "foobar"} nilCb := func(rd io.Reader) error { return nil } return be.Load(context.TODO(), h, 10, 0, nilCb) } @@ -155,13 +156,13 @@ func TestConcurrencyLimitLoad(t *testing.T) { func TestConcurrencyLimitStat(t *testing.T) { wait, unblock := countingBlocker() concurrencyTester(t, func(m *mock.Backend) { - m.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { + m.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { wait() - return restic.FileInfo{}, nil + return backend.FileInfo{}, nil } - }, func(be restic.Backend) func() error { + }, func(be backend.Backend) func() error { return func() error { - h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + h := backend.Handle{Type: backend.PackFile, Name: "foobar"} _, err := be.Stat(context.TODO(), h) return err } @@ -171,13 +172,13 @@ func TestConcurrencyLimitStat(t *testing.T) { func TestConcurrencyLimitDelete(t *testing.T) { wait, unblock := countingBlocker() concurrencyTester(t, func(m *mock.Backend) { - m.RemoveFn = func(ctx context.Context, h restic.Handle) error { + m.RemoveFn = func(ctx context.Context, h backend.Handle) error { wait() return nil } - }, func(be restic.Backend) func() error { + }, func(be backend.Backend) func() error { return func() error { - h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + h := backend.Handle{Type: backend.PackFile, Name: "foobar"} return be.Remove(context.TODO(), h) } }, unblock, false) @@ -186,14 +187,49 @@ func TestConcurrencyLimitDelete(t *testing.T) { func TestConcurrencyUnlimitedLockSave(t *testing.T) { wait, unblock := countingBlocker() concurrencyTester(t, func(m *mock.Backend) { - m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { wait() return nil } - }, func(be restic.Backend) func() error { + }, func(be backend.Backend) func() error { return func() error { - h := restic.Handle{Type: restic.LockFile, Name: "foobar"} + h := backend.Handle{Type: backend.LockFile, Name: "foobar"} return be.Save(context.TODO(), h, nil) } }, unblock, true) } + +func TestFreeze(t *testing.T) { + var counter int64 + m := mock.NewBackend() + m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { + atomic.AddInt64(&counter, 1) + return nil + } + m.ConnectionsFn = func() uint { return 2 } + be := sema.NewBackend(m) + fb := be.(backend.FreezeBackend) + + // Freeze backend + fb.Freeze() + + // Start Save call that should block + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + h := backend.Handle{Type: backend.PackFile, Name: "foobar"} + test.OK(t, be.Save(context.TODO(), h, nil)) + }() + + // check + time.Sleep(1 * time.Millisecond) + val := atomic.LoadInt64(&counter) + test.Assert(t, val == 0, "save call worked despite frozen backend") + + // unfreeze and check that save did complete + fb.Unfreeze() + wg.Wait() + val = atomic.LoadInt64(&counter) + test.Assert(t, val == 1, "save call should have completed") +} diff --git a/internal/backend/sftp/config.go b/internal/backend/sftp/config.go index ed7c2cafa..65af50d19 100644 --- a/internal/backend/sftp/config.go +++ b/internal/backend/sftp/config.go @@ -13,8 +13,9 @@ import ( type Config struct { User, Host, Port, Path string - Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"` + Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"` Command string `option:"command" help:"specify command to create sftp connection"` + Args string `option:"args" help:"specify arguments for ssh"` Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"` } diff --git a/internal/backend/sftp/layout_test.go b/internal/backend/sftp/layout_test.go index fc8d80928..9cf24a753 100644 --- a/internal/backend/sftp/layout_test.go +++ b/internal/backend/sftp/layout_test.go @@ -6,8 +6,8 @@ import ( "path/filepath" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/sftp" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -56,7 +56,7 @@ func TestLayout(t *testing.T) { } packs := make(map[string]bool) - err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { packs[fi.Name] = false return nil }) diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index 3e127ef05..0a94e4aa3 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -17,9 +17,9 @@ import ( "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/limiter" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" "github.com/cenkalti/backoff/v4" "github.com/pkg/sftp" @@ -38,10 +38,10 @@ type SFTP struct { layout.Layout Config - backend.Modes + util.Modes } -var _ restic.Backend = &SFTP{} +var _ backend.Backend = &SFTP{} func NewFactory() location.Factory { return location.NewLimitedBackendFactory("sftp", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open)) @@ -83,9 +83,9 @@ func startClient(cfg Config) (*SFTP, error) { return nil, errors.Wrap(err, "cmd.StdoutPipe") } - bg, err := backend.StartForeground(cmd) + bg, err := util.StartForeground(cmd) if err != nil { - if backend.IsErrDot(err) { + if util.IsErrDot(err) { return nil, errors.Errorf("cannot implicitly run relative executable %v found in current directory, use -o sftp.command=./ to override", cmd.Path) } return nil, err @@ -152,8 +152,8 @@ func open(ctx context.Context, sftp *SFTP, cfg Config) (*SFTP, error) { debug.Log("layout: %v\n", sftp.Layout) - fi, err := sftp.c.Stat(sftp.Layout.Filename(restic.Handle{Type: restic.ConfigFile})) - m := backend.DeriveModesFromFileInfo(fi, err) + fi, err := sftp.c.Stat(sftp.Layout.Filename(backend.Handle{Type: backend.ConfigFile})) + m := util.DeriveModesFromFileInfo(fi, err) debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir) sftp.Config = cfg @@ -213,6 +213,9 @@ func buildSSHCommand(cfg Config) (cmd string, args []string, err error) { if err != nil { return "", nil, err } + if cfg.Args != "" { + return "", nil, errors.New("cannot specify both sftp.command and sftp.args options") + } return args[0], args[1:], nil } @@ -226,11 +229,19 @@ func buildSSHCommand(cfg Config) (cmd string, args []string, err error) { args = append(args, "-p", port) } if cfg.User != "" { - args = append(args, "-l") - args = append(args, cfg.User) + args = append(args, "-l", cfg.User) } - args = append(args, "-s") - args = append(args, "sftp") + + if cfg.Args != "" { + a, err := backend.SplitShellStrings(cfg.Args) + if err != nil { + return "", nil, err + } + + args = append(args, a...) + } + + args = append(args, "-s", "sftp") return cmd, args, nil } @@ -248,10 +259,10 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) { return nil, err } - sftp.Modes = backend.DefaultModes + sftp.Modes = util.DefaultModes // test if config file already exists - _, err = sftp.c.Lstat(sftp.Layout.Filename(restic.Handle{Type: restic.ConfigFile})) + _, err = sftp.c.Lstat(sftp.Layout.Filename(backend.Handle{Type: backend.ConfigFile})) if err == nil { return nil, errors.New("config file already exists") } @@ -291,7 +302,7 @@ func Join(parts ...string) string { } // tempSuffix generates a random string suffix that should be sufficiently long -// to avoid accidential conflicts +// to avoid accidental conflicts func tempSuffix() string { var nonce [16]byte _, err := rand.Read(nonce[:]) @@ -302,7 +313,7 @@ func tempSuffix() string { } // Save stores data in the backend at the handle. -func (r *SFTP) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) error { +func (r *SFTP) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) error { if err := r.clientError(); err != nil { return err } @@ -402,11 +413,11 @@ func (r *SFTP) checkNoSpace(dir string, size int64, origErr error) error { // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (r *SFTP) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - return backend.DefaultLoad(ctx, h, length, offset, r.openReader, fn) +func (r *SFTP) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + return util.DefaultLoad(ctx, h, length, offset, r.openReader, fn) } -func (r *SFTP) openReader(_ context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (r *SFTP) openReader(_ context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { f, err := r.c.Open(r.Filename(h)) if err != nil { return nil, err @@ -430,21 +441,21 @@ func (r *SFTP) openReader(_ context.Context, h restic.Handle, length int, offset } // Stat returns information about a blob. -func (r *SFTP) Stat(_ context.Context, h restic.Handle) (restic.FileInfo, error) { +func (r *SFTP) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) { if err := r.clientError(); err != nil { - return restic.FileInfo{}, err + return backend.FileInfo{}, err } fi, err := r.c.Lstat(r.Filename(h)) if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "Lstat") + return backend.FileInfo{}, errors.Wrap(err, "Lstat") } - return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil + return backend.FileInfo{Size: fi.Size(), Name: h.Name}, nil } // Remove removes the content stored at name. -func (r *SFTP) Remove(_ context.Context, h restic.Handle) error { +func (r *SFTP) Remove(_ context.Context, h backend.Handle) error { if err := r.clientError(); err != nil { return err } @@ -454,7 +465,7 @@ func (r *SFTP) Remove(_ context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (r *SFTP) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (r *SFTP) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { basedir, subdirs := r.Basedir(t) walker := r.c.Walk(basedir) for { @@ -487,7 +498,7 @@ func (r *SFTP) List(ctx context.Context, t restic.FileType, fn func(restic.FileI debug.Log("send %v\n", path.Base(walker.Path())) - rfi := restic.FileInfo{ + rfi := backend.FileInfo{ Name: path.Base(walker.Path()), Size: fi.Size(), } diff --git a/internal/backend/sftp/sshcmd_test.go b/internal/backend/sftp/sshcmd_test.go index 822f28b5d..601c94bd2 100644 --- a/internal/backend/sftp/sshcmd_test.go +++ b/internal/backend/sftp/sshcmd_test.go @@ -9,38 +9,57 @@ var sshcmdTests = []struct { cfg Config cmd string args []string + err string }{ { Config{User: "user", Host: "host", Path: "dir/subdir"}, "ssh", []string{"host", "-l", "user", "-s", "sftp"}, + "", }, { Config{Host: "host", Path: "dir/subdir"}, "ssh", []string{"host", "-s", "sftp"}, + "", }, { Config{Host: "host", Port: "10022", Path: "/dir/subdir"}, "ssh", []string{"host", "-p", "10022", "-s", "sftp"}, + "", }, { Config{User: "user", Host: "host", Port: "10022", Path: "/dir/subdir"}, "ssh", []string{"host", "-p", "10022", "-l", "user", "-s", "sftp"}, + "", + }, + { + Config{User: "user", Host: "host", Port: "10022", Path: "/dir/subdir", Args: "-i /path/to/id_rsa"}, + "ssh", + []string{"host", "-p", "10022", "-l", "user", "-i", "/path/to/id_rsa", "-s", "sftp"}, + "", + }, + { + Config{Command: "ssh something", Args: "-i /path/to/id_rsa"}, + "", + nil, + "cannot specify both sftp.command and sftp.args options", }, { // IPv6 address. Config{User: "user", Host: "::1", Path: "dir"}, "ssh", []string{"::1", "-l", "user", "-s", "sftp"}, + "", }, { // IPv6 address with zone and port. Config{User: "user", Host: "::1%lo0", Port: "22", Path: "dir"}, "ssh", []string{"::1%lo0", "-p", "22", "-l", "user", "-s", "sftp"}, + "", }, } @@ -48,8 +67,14 @@ func TestBuildSSHCommand(t *testing.T) { for i, test := range sshcmdTests { t.Run("", func(t *testing.T) { cmd, args, err := buildSSHCommand(test.cfg) - if err != nil { - t.Fatalf("%v in test %d", err, i) + if test.err != "" { + if err.Error() != test.err { + t.Fatalf("expected error %v got %v", test.err, err.Error()) + } + } else { + if err != nil { + t.Fatalf("%v in test %d", err, i) + } } if cmd != test.cmd { diff --git a/internal/backend/shell_split.go b/internal/backend/shell_split.go index eff527616..888c993a0 100644 --- a/internal/backend/shell_split.go +++ b/internal/backend/shell_split.go @@ -6,7 +6,7 @@ import ( "github.com/restic/restic/internal/errors" ) -// shellSplitter splits a command string into separater arguments. It supports +// shellSplitter splits a command string into separated arguments. It supports // single and double quoted strings. type shellSplitter struct { quote rune diff --git a/internal/backend/smb/smb.go b/internal/backend/smb/smb.go index 493e02d9e..367069c59 100644 --- a/internal/backend/smb/smb.go +++ b/internal/backend/smb/smb.go @@ -19,6 +19,7 @@ import ( "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/limiter" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -49,7 +50,7 @@ import ( type Backend struct { Config layout.Layout - backend.Modes + util.Modes sessions int32 poolMu sync.Mutex @@ -58,7 +59,7 @@ type Backend struct { } // make sure that *Backend implements backend.Backend -var _ restic.Backend = &Backend{} +var _ backend.Backend = &Backend{} func NewFactory() location.Factory { return location.NewLimitedBackendFactory("smb", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open)) @@ -93,8 +94,8 @@ func open(ctx context.Context, cfg Config) (*Backend, error) { } defer b.putConnection(cn) - stat, err := cn.smbShare.Stat(l.Filename(restic.Handle{Type: restic.ConfigFile})) - m := backend.DeriveModesFromFileInfo(stat, err) + stat, err := cn.smbShare.Stat(l.Filename(backend.Handle{Type: restic.ConfigFile})) + m := util.DeriveModesFromFileInfo(stat, err) debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir) b.Modes = m @@ -125,7 +126,7 @@ func Create(ctx context.Context, cfg Config) (*Backend, error) { defer b.putConnection(cn) // test if config file already exists - _, err = cn.smbShare.Lstat(b.Filename(restic.Handle{Type: restic.ConfigFile})) + _, err = cn.smbShare.Lstat(b.Filename(backend.Handle{Type: restic.ConfigFile})) if err == nil { return nil, errors.New("config file already exists") } @@ -171,7 +172,7 @@ func (b *Backend) Join(p ...string) string { } // Save stores data in the backend at the handle. -func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) (err error) { +func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) (err error) { filename := b.Filename(h) tmpFilename := filename + "-restic-temp-" + tempSuffix() dir := filepath.Dir(tmpFilename) @@ -267,11 +268,11 @@ func (cn *conn) setFileReadonly(f string, mode os.FileMode) error { // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - return backend.DefaultLoad(ctx, h, length, offset, b.openReader, fn) +func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + return util.DefaultLoad(ctx, h, length, offset, b.openReader, fn) } -func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (b *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { b.addSession() // Show session in use defer b.removeSession() cn, err := b.getConnection(ctx, b.ShareName) @@ -301,23 +302,23 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o } // Stat returns information about a blob. -func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { cn, err := b.getConnection(ctx, b.ShareName) if err != nil { - return restic.FileInfo{}, err + return backend.FileInfo{}, err } defer b.putConnection(cn) fi, err := cn.smbShare.Stat(b.Filename(h)) if err != nil { - return restic.FileInfo{}, errors.WithStack(err) + return backend.FileInfo{}, errors.WithStack(err) } - return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil + return backend.FileInfo{Size: fi.Size(), Name: h.Name}, nil } // Remove removes the blob with the given name and type. -func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (b *Backend) Remove(ctx context.Context, h backend.Handle) error { fn := b.Filename(h) cn, err := b.getConnection(ctx, b.ShareName) @@ -337,7 +338,7 @@ func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) (err error) { +func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(backend.FileInfo) error) (err error) { cn, err := b.getConnection(ctx, b.ShareName) if err != nil { return err @@ -363,7 +364,7 @@ func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi // two levels of directory structure (including dir itself as the first level). // Also, visitDirs assumes it sees a directory full of directories, while // visitFiles wants a directory full or regular files. -func (b *Backend) visitDirs(ctx context.Context, cn *conn, dir string, fn func(restic.FileInfo) error) error { +func (b *Backend) visitDirs(ctx context.Context, cn *conn, dir string, fn func(backend.FileInfo) error) error { d, err := cn.smbShare.Open(dir) if err != nil { return err @@ -390,7 +391,7 @@ func (b *Backend) visitDirs(ctx context.Context, cn *conn, dir string, fn func(r return ctx.Err() } -func (b *Backend) visitFiles(ctx context.Context, cn *conn, dir string, fn func(restic.FileInfo) error, ignoreNotADirectory bool) error { +func (b *Backend) visitFiles(ctx context.Context, cn *conn, dir string, fn func(backend.FileInfo) error, ignoreNotADirectory bool) error { d, err := cn.smbShare.Open(dir) if err != nil { return err @@ -424,7 +425,7 @@ func (b *Backend) visitFiles(ctx context.Context, cn *conn, dir string, fn func( default: } - err := fn(restic.FileInfo{ + err := fn(backend.FileInfo{ Name: fi.Name(), Size: fi.Size(), }) diff --git a/internal/backend/swift/config.go b/internal/backend/swift/config.go index 5be2d9ce0..9adb80522 100644 --- a/internal/backend/swift/config.go +++ b/internal/backend/swift/config.go @@ -4,9 +4,9 @@ import ( "os" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" ) // Config contains basic configuration needed to specify swift location for a swift server @@ -74,7 +74,7 @@ func ParseConfig(s string) (*Config, error) { return &cfg, nil } -var _ restic.ApplyEnvironmenter = &Config{} +var _ backend.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. func (cfg *Config) ApplyEnvironment(prefix string) { diff --git a/internal/backend/swift/swift.go b/internal/backend/swift/swift.go index 1cfc0a65b..6943f0180 100644 --- a/internal/backend/swift/swift.go +++ b/internal/backend/swift/swift.go @@ -16,9 +16,9 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" "github.com/ncw/swift/v2" ) @@ -32,8 +32,8 @@ type beSwift struct { layout.Layout } -// ensure statically that *beSwift implements restic.Backend. -var _ restic.Backend = &beSwift{} +// ensure statically that *beSwift implements backend.Backend. +var _ backend.Backend = &beSwift{} func NewFactory() location.Factory { return location.NewHTTPBackendFactory("swift", ParseConfig, location.NoPassword, Open, Open) @@ -41,7 +41,7 @@ func NewFactory() location.Factory { // Open opens the swift backend at a container in region. The container is // created if it does not exist yet. -func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { debug.Log("config %#v", cfg) be := &beSwift{ @@ -134,11 +134,11 @@ func (be *beSwift) HasAtomicReplace() bool { // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *beSwift) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn) +func (be *beSwift) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } -func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *beSwift) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { objName := be.Filename(h) @@ -160,7 +160,7 @@ func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int, } // Save stores data in the backend at the handle. -func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *beSwift) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { objName := be.Filename(h) encoding := "binary/octet-stream" @@ -174,19 +174,19 @@ func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe } // Stat returns information about a blob. -func (be *beSwift) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { +func (be *beSwift) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) { objName := be.Filename(h) obj, _, err := be.conn.Object(ctx, be.container, objName) if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "conn.Object") + return backend.FileInfo{}, errors.Wrap(err, "conn.Object") } - return restic.FileInfo{Size: obj.Bytes, Name: h.Name}, nil + return backend.FileInfo{Size: obj.Bytes, Name: h.Name}, nil } // Remove removes the blob with the given name and type. -func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error { +func (be *beSwift) Remove(ctx context.Context, h backend.Handle) error { objName := be.Filename(h) err := be.conn.ObjectDelete(ctx, be.container, objName) @@ -195,7 +195,7 @@ func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (be *beSwift) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *beSwift) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { prefix, _ := be.Basedir(t) prefix += "/" @@ -212,7 +212,7 @@ func (be *beSwift) List(ctx context.Context, t restic.FileType, fn func(restic.F continue } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: m, Size: obj.Bytes, } @@ -245,7 +245,7 @@ func (be *beSwift) IsNotExist(err error) bool { // Delete removes all restic objects in the container. // It will not remove the container itself. func (be *beSwift) Delete(ctx context.Context) error { - return backend.DefaultDelete(ctx, be) + return util.DefaultDelete(ctx, be) } // Close does nothing diff --git a/internal/backend/swift/swift_test.go b/internal/backend/swift/swift_test.go index 98ee5b1c1..355947cc7 100644 --- a/internal/backend/swift/swift_test.go +++ b/internal/backend/swift/swift_test.go @@ -6,9 +6,9 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/swift" "github.com/restic/restic/internal/backend/test" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -20,7 +20,7 @@ func newSwiftTestSuite(t testing.TB) *test.Suite[swift.Config] { // wait for removals for at least 5m WaitForDelayedRemoval: 5 * time.Minute, - ErrorHandler: func(t testing.TB, be restic.Backend, err error) error { + ErrorHandler: func(t testing.TB, be backend.Backend, err error) error { if err == nil { return nil } diff --git a/internal/backend/test/benchmarks.go b/internal/backend/test/benchmarks.go index 150ef3987..e4271a386 100644 --- a/internal/backend/test/benchmarks.go +++ b/internal/backend/test/benchmarks.go @@ -6,22 +6,23 @@ import ( "io" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) -func saveRandomFile(t testing.TB, be restic.Backend, length int) ([]byte, restic.Handle) { +func saveRandomFile(t testing.TB, be backend.Backend, length int) ([]byte, backend.Handle) { data := test.Random(23, length) id := restic.Hash(data) - handle := restic.Handle{Type: restic.PackFile, Name: id.String()} - err := be.Save(context.TODO(), handle, restic.NewByteReader(data, be.Hasher())) + handle := backend.Handle{Type: backend.PackFile, Name: id.String()} + err := be.Save(context.TODO(), handle, backend.NewByteReader(data, be.Hasher())) if err != nil { t.Fatalf("Save() error: %+v", err) } return data, handle } -func remove(t testing.TB, be restic.Backend, h restic.Handle) { +func remove(t testing.TB, be backend.Backend, h backend.Handle) { if err := be.Remove(context.TODO(), h); err != nil { t.Fatalf("Remove() returned error: %v", err) } @@ -146,9 +147,9 @@ func (s *Suite[C]) BenchmarkSave(t *testing.B) { length := 1<<24 + 2123 data := test.Random(23, length) id := restic.Hash(data) - handle := restic.Handle{Type: restic.PackFile, Name: id.String()} + handle := backend.Handle{Type: backend.PackFile, Name: id.String()} - rd := restic.NewByteReader(data, be.Hasher()) + rd := backend.NewByteReader(data, be.Hasher()) t.SetBytes(int64(length)) t.ResetTimer() diff --git a/internal/backend/test/doc.go b/internal/backend/test/doc.go index 25bdf0417..c15ed4d82 100644 --- a/internal/backend/test/doc.go +++ b/internal/backend/test/doc.go @@ -17,7 +17,7 @@ // // func newTestSuite(t testing.TB) *test.Suite { // return &test.Suite{ -// Create: func(cfg interface{}) (restic.Backend, error) { +// Create: func(cfg interface{}) (backend.Backend, error) { // [...] // }, // [...] diff --git a/internal/backend/test/suite.go b/internal/backend/test/suite.go index bb77124d7..ad8eb4c5d 100644 --- a/internal/backend/test/suite.go +++ b/internal/backend/test/suite.go @@ -11,7 +11,6 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) @@ -35,7 +34,7 @@ type Suite[C any] struct { WaitForDelayedRemoval time.Duration // ErrorHandler allows ignoring certain errors. - ErrorHandler func(testing.TB, restic.Backend, error) error + ErrorHandler func(testing.TB, backend.Backend, error) error } // RunTests executes all defined tests as subtests of t. @@ -156,7 +155,7 @@ func (s *Suite[C]) RunBenchmarks(b *testing.B) { s.cleanup(b) } -func (s *Suite[C]) createOrError() (restic.Backend, error) { +func (s *Suite[C]) createOrError() (backend.Backend, error) { tr, err := backend.Transport(backend.TransportOptions{}) if err != nil { return nil, fmt.Errorf("cannot create transport for tests: %v", err) @@ -167,7 +166,7 @@ func (s *Suite[C]) createOrError() (restic.Backend, error) { return nil, err } - _, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile}) + _, err = be.Stat(context.TODO(), backend.Handle{Type: backend.ConfigFile}) if err != nil && !be.IsNotExist(err) { return nil, err } @@ -179,7 +178,7 @@ func (s *Suite[C]) createOrError() (restic.Backend, error) { return be, nil } -func (s *Suite[C]) create(t testing.TB) restic.Backend { +func (s *Suite[C]) create(t testing.TB) backend.Backend { be, err := s.createOrError() if err != nil { t.Fatal(err) @@ -187,7 +186,7 @@ func (s *Suite[C]) create(t testing.TB) restic.Backend { return be } -func (s *Suite[C]) open(t testing.TB) restic.Backend { +func (s *Suite[C]) open(t testing.TB) backend.Backend { tr, err := backend.Transport(backend.TransportOptions{}) if err != nil { t.Fatalf("cannot create transport for tests: %v", err) @@ -208,7 +207,7 @@ func (s *Suite[C]) cleanup(t testing.TB) { s.close(t, be) } -func (s *Suite[C]) close(t testing.TB, be restic.Backend) { +func (s *Suite[C]) close(t testing.TB, be backend.Backend) { err := be.Close() if err != nil { t.Fatal(err) diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go index c2e5d0fc0..414bf1c3b 100644 --- a/internal/backend/test/tests.go +++ b/internal/backend/test/tests.go @@ -27,7 +27,7 @@ func seedRand(t testing.TB) { t.Logf("rand initialized with seed %d", seed) } -func beTest(ctx context.Context, be restic.Backend, h restic.Handle) (bool, error) { +func beTest(ctx context.Context, be backend.Backend, h backend.Handle) (bool, error) { _, err := be.Stat(ctx, h) if err != nil && be.IsNotExist(err) { return false, nil @@ -49,7 +49,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) { defer s.close(t, b) // remove a config if present - cfgHandle := restic.Handle{Type: restic.ConfigFile} + cfgHandle := backend.Handle{Type: backend.ConfigFile} cfgPresent, err := beTest(context.TODO(), b, cfgHandle) if err != nil { t.Fatalf("unable to test for config: %+v", err) @@ -60,7 +60,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) { } // save a config - store(t, b, restic.ConfigFile, []byte("test config")) + store(t, b, backend.ConfigFile, []byte("test config")) // now create the backend again, this must fail _, err = s.createOrError() @@ -69,7 +69,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) { } // remove config - err = b.Remove(context.TODO(), restic.Handle{Type: restic.ConfigFile, Name: ""}) + err = b.Remove(context.TODO(), backend.Handle{Type: backend.ConfigFile, Name: ""}) if err != nil { t.Fatalf("unexpected error removing config: %+v", err) } @@ -94,13 +94,13 @@ func (s *Suite[C]) TestConfig(t *testing.T) { var testString = "Config" // create config and read it back - _, err := backend.LoadAll(context.TODO(), nil, b, restic.Handle{Type: restic.ConfigFile}) + _, err := backend.LoadAll(context.TODO(), nil, b, backend.Handle{Type: backend.ConfigFile}) if err == nil { t.Fatalf("did not get expected error for non-existing config") } test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize error from LoadAll(): %v", err) - err = b.Save(context.TODO(), restic.Handle{Type: restic.ConfigFile}, restic.NewByteReader([]byte(testString), b.Hasher())) + err = b.Save(context.TODO(), backend.Handle{Type: backend.ConfigFile}, backend.NewByteReader([]byte(testString), b.Hasher())) if err != nil { t.Fatalf("Save() error: %+v", err) } @@ -108,7 +108,7 @@ func (s *Suite[C]) TestConfig(t *testing.T) { // try accessing the config with different names, should all return the // same config for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} { - h := restic.Handle{Type: restic.ConfigFile, Name: name} + h := backend.Handle{Type: backend.ConfigFile, Name: name} buf, err := backend.LoadAll(context.TODO(), nil, b, h) if err != nil { t.Fatalf("unable to read config with name %q: %+v", name, err) @@ -120,7 +120,7 @@ func (s *Suite[C]) TestConfig(t *testing.T) { } // remove the config - remove(t, b, restic.Handle{Type: restic.ConfigFile}) + remove(t, b, backend.Handle{Type: backend.ConfigFile}) } // TestLoad tests the backend's Load function. @@ -130,7 +130,7 @@ func (s *Suite[C]) TestLoad(t *testing.T) { b := s.open(t) defer s.close(t, b) - err := testLoad(b, restic.Handle{Type: restic.PackFile, Name: "foobar"}) + err := testLoad(b, backend.Handle{Type: backend.PackFile, Name: "foobar"}) if err == nil { t.Fatalf("Load() did not return an error for non-existing blob") } @@ -141,8 +141,8 @@ func (s *Suite[C]) TestLoad(t *testing.T) { data := test.Random(23, length) id := restic.Hash(data) - handle := restic.Handle{Type: restic.PackFile, Name: id.String()} - err = b.Save(context.TODO(), handle, restic.NewByteReader(data, b.Hasher())) + handle := backend.Handle{Type: backend.PackFile, Name: id.String()} + err = b.Save(context.TODO(), handle, backend.NewByteReader(data, b.Hasher())) if err != nil { t.Fatalf("Save() error: %+v", err) } @@ -243,7 +243,7 @@ func (s *Suite[C]) TestList(t *testing.T) { // Check that the backend is empty to start with var found []string - err := b.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { found = append(found, fi.Name) return nil }) @@ -259,8 +259,8 @@ func (s *Suite[C]) TestList(t *testing.T) { for i := 0; i < numTestFiles; i++ { data := test.Random(rand.Int(), rand.Intn(100)+55) id := restic.Hash(data) - h := restic.Handle{Type: restic.PackFile, Name: id.String()} - err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher())) + h := backend.Handle{Type: backend.PackFile, Name: id.String()} + err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher())) if err != nil { t.Fatal(err) } @@ -284,7 +284,7 @@ func (s *Suite[C]) TestList(t *testing.T) { s.SetListMaxItems(test.maxItems) } - err := b.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { id, err := restic.ParseID(fi.Name) if err != nil { t.Fatal(err) @@ -320,9 +320,9 @@ func (s *Suite[C]) TestList(t *testing.T) { } t.Logf("remove %d files", numTestFiles) - handles := make([]restic.Handle, 0, len(list1)) + handles := make([]backend.Handle, 0, len(list1)) for id := range list1 { - handles = append(handles, restic.Handle{Type: restic.PackFile, Name: id.String()}) + handles = append(handles, backend.Handle{Type: backend.PackFile, Name: id.String()}) } err = s.delayedRemove(t, b, handles...) @@ -340,13 +340,13 @@ func (s *Suite[C]) TestListCancel(t *testing.T) { b := s.open(t) defer s.close(t, b) - testFiles := make([]restic.Handle, 0, numTestFiles) + testFiles := make([]backend.Handle, 0, numTestFiles) for i := 0; i < numTestFiles; i++ { data := []byte(fmt.Sprintf("random test blob %v", i)) id := restic.Hash(data) - h := restic.Handle{Type: restic.PackFile, Name: id.String()} - err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher())) + h := backend.Handle{Type: backend.PackFile, Name: id.String()} + err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher())) if err != nil { t.Fatal(err) } @@ -358,7 +358,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) { cancel() // pass in a cancelled context - err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error { + err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error { t.Errorf("got FileInfo %v for cancelled context", fi) return nil }) @@ -373,7 +373,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) { defer cancel() i := 0 - err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error { + err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error { i++ // cancel the context on the first file if i == 1 { @@ -396,7 +396,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) { defer cancel() i := 0 - err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error { + err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error { // cancel the context at the last file i++ if i == numTestFiles { @@ -423,11 +423,16 @@ func (s *Suite[C]) TestListCancel(t *testing.T) { i := 0 // pass in a context with a timeout - err := b.List(ctxTimeout, restic.PackFile, func(fi restic.FileInfo) error { + err := b.List(ctxTimeout, backend.PackFile, func(fi backend.FileInfo) error { i++ // wait until the context is cancelled <-ctxTimeout.Done() + // The cancellation of a context first closes the done channel of the context and + // _afterwards_ propagates the cancellation to child contexts. If the List + // implementation uses a child context, then it may take a moment until that context + // is also cancelled. Thus give the context cancellation a moment to propagate. + time.Sleep(time.Millisecond) return nil }) @@ -489,11 +494,11 @@ func (s *Suite[C]) TestSave(t *testing.T) { data := test.Random(23, length) id = sha256.Sum256(data) - h := restic.Handle{ - Type: restic.PackFile, + h := backend.Handle{ + Type: backend.PackFile, Name: id.String(), } - err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher())) + err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher())) test.OK(t, err) buf, err := backend.LoadAll(context.TODO(), nil, b, h) @@ -541,7 +546,7 @@ func (s *Suite[C]) TestSave(t *testing.T) { t.Fatal(err) } - h := restic.Handle{Type: restic.PackFile, Name: id.String()} + h := backend.Handle{Type: backend.PackFile, Name: id.String()} // wrap the tempfile in an errorCloser, so we can detect if the backend // closes the reader @@ -580,7 +585,7 @@ func (s *Suite[C]) TestSave(t *testing.T) { } type incompleteByteReader struct { - restic.ByteReader + backend.ByteReader } func (r *incompleteByteReader) Length() int64 { @@ -604,8 +609,8 @@ func (s *Suite[C]) TestSaveError(t *testing.T) { copy(id[:], data) // test that incomplete uploads fail - h := restic.Handle{Type: restic.PackFile, Name: id.String()} - err := b.Save(context.TODO(), h, &incompleteByteReader{ByteReader: *restic.NewByteReader(data, b.Hasher())}) + h := backend.Handle{Type: backend.PackFile, Name: id.String()} + err := b.Save(context.TODO(), h, &incompleteByteReader{ByteReader: *backend.NewByteReader(data, b.Hasher())}) // try to delete possible leftovers _ = s.delayedRemove(t, b, h) if err == nil { @@ -614,7 +619,7 @@ func (s *Suite[C]) TestSaveError(t *testing.T) { } type wrongByteReader struct { - restic.ByteReader + backend.ByteReader } func (b *wrongByteReader) Hash() []byte { @@ -643,8 +648,8 @@ func (s *Suite[C]) TestSaveWrongHash(t *testing.T) { copy(id[:], data) // test that upload with hash mismatch fails - h := restic.Handle{Type: restic.PackFile, Name: id.String()} - err := b.Save(context.TODO(), h, &wrongByteReader{ByteReader: *restic.NewByteReader(data, b.Hasher())}) + h := backend.Handle{Type: backend.PackFile, Name: id.String()} + err := b.Save(context.TODO(), h, &wrongByteReader{ByteReader: *backend.NewByteReader(data, b.Hasher())}) exists, err2 := beTest(context.TODO(), b, h) if err2 != nil { t.Fatal(err2) @@ -669,23 +674,23 @@ var testStrings = []struct { {"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"}, } -func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) restic.Handle { +func store(t testing.TB, b backend.Backend, tpe backend.FileType, data []byte) backend.Handle { id := restic.Hash(data) - h := restic.Handle{Name: id.String(), Type: tpe} - err := b.Save(context.TODO(), h, restic.NewByteReader([]byte(data), b.Hasher())) + h := backend.Handle{Name: id.String(), Type: tpe} + err := b.Save(context.TODO(), h, backend.NewByteReader([]byte(data), b.Hasher())) test.OK(t, err) return h } // testLoad loads a blob (but discards its contents). -func testLoad(b restic.Backend, h restic.Handle) error { +func testLoad(b backend.Backend, h backend.Handle) error { return b.Load(context.TODO(), h, 0, 0, func(rd io.Reader) (ierr error) { _, ierr = io.Copy(io.Discard, rd) return ierr }) } -func (s *Suite[C]) delayedRemove(t testing.TB, be restic.Backend, handles ...restic.Handle) error { +func (s *Suite[C]) delayedRemove(t testing.TB, be backend.Backend, handles ...backend.Handle) error { // Some backend (swift, I'm looking at you) may implement delayed // removal of data. Let's wait a bit if this happens. @@ -729,11 +734,11 @@ func (s *Suite[C]) delayedRemove(t testing.TB, be restic.Backend, handles ...res return nil } -func delayedList(t testing.TB, b restic.Backend, tpe restic.FileType, max int, maxwait time.Duration) restic.IDs { +func delayedList(t testing.TB, b backend.Backend, tpe backend.FileType, max int, maxwait time.Duration) restic.IDs { list := restic.NewIDSet() start := time.Now() for i := 0; i < max; i++ { - err := b.List(context.TODO(), tpe, func(fi restic.FileInfo) error { + err := b.List(context.TODO(), tpe, func(fi backend.FileInfo) error { id := restic.TestParseID(fi.Name) list.Insert(id) return nil @@ -758,9 +763,9 @@ func (s *Suite[C]) TestBackend(t *testing.T) { test.Assert(t, !b.IsNotExist(nil), "IsNotExist() recognized nil error") - for _, tpe := range []restic.FileType{ - restic.PackFile, restic.KeyFile, restic.LockFile, - restic.SnapshotFile, restic.IndexFile, + for _, tpe := range []backend.FileType{ + backend.PackFile, backend.KeyFile, backend.LockFile, + backend.SnapshotFile, backend.IndexFile, } { // detect non-existing files for _, ts := range testStrings { @@ -768,7 +773,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) { test.OK(t, err) // test if blob is already in repository - h := restic.Handle{Type: tpe, Name: id.String()} + h := backend.Handle{Type: tpe, Name: id.String()} ret, err := beTest(context.TODO(), b, h) test.OK(t, err) test.Assert(t, !ret, "blob was found to exist before creating") @@ -794,7 +799,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) { store(t, b, tpe, []byte(ts.data)) // test Load() - h := restic.Handle{Type: tpe, Name: ts.id} + h := backend.Handle{Type: tpe, Name: ts.id} buf, err := backend.LoadAll(context.TODO(), nil, b, h) test.OK(t, err) test.Equals(t, ts.data, string(buf)) @@ -818,7 +823,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) { // test adding the first file again ts := testStrings[0] - h := restic.Handle{Type: tpe, Name: ts.id} + h := backend.Handle{Type: tpe, Name: ts.id} // remove and recreate err := s.delayedRemove(t, b, h) @@ -830,7 +835,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) { test.Assert(t, !ok, "removed blob still present") // create blob - err = b.Save(context.TODO(), h, restic.NewByteReader([]byte(ts.data), b.Hasher())) + err = b.Save(context.TODO(), h, backend.NewByteReader([]byte(ts.data), b.Hasher())) test.OK(t, err) // list items @@ -854,12 +859,12 @@ func (s *Suite[C]) TestBackend(t *testing.T) { t.Fatalf("lists aren't equal, want:\n %v\n got:\n%v\n", IDs, list) } - var handles []restic.Handle + var handles []backend.Handle for _, ts := range testStrings { id, err := restic.ParseID(ts.id) test.OK(t, err) - h := restic.Handle{Type: tpe, Name: id.String()} + h := backend.Handle{Type: tpe, Name: id.String()} found, err := beTest(context.TODO(), b, h) test.OK(t, err) diff --git a/internal/backend/util/defaults.go b/internal/backend/util/defaults.go new file mode 100644 index 000000000..e5b6fc456 --- /dev/null +++ b/internal/backend/util/defaults.go @@ -0,0 +1,50 @@ +package util + +import ( + "context" + "io" + + "github.com/restic/restic/internal/backend" +) + +// DefaultLoad implements Backend.Load using lower-level openReader func +func DefaultLoad(ctx context.Context, h backend.Handle, length int, offset int64, + openReader func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error), + fn func(rd io.Reader) error) error { + + rd, err := openReader(ctx, h, length, offset) + if err != nil { + return err + } + err = fn(rd) + if err != nil { + _ = rd.Close() // ignore secondary errors closing the reader + return err + } + return rd.Close() +} + +// DefaultDelete removes all restic keys in the bucket. It will not remove the bucket itself. +func DefaultDelete(ctx context.Context, be backend.Backend) error { + alltypes := []backend.FileType{ + backend.PackFile, + backend.KeyFile, + backend.LockFile, + backend.SnapshotFile, + backend.IndexFile} + + for _, t := range alltypes { + err := be.List(ctx, t, func(fi backend.FileInfo) error { + return be.Remove(ctx, backend.Handle{Type: t, Name: fi.Name}) + }) + if err != nil { + return nil + } + } + err := be.Remove(ctx, backend.Handle{Type: backend.ConfigFile}) + if err != nil && be.IsNotExist(err) { + err = nil + } + + return err +} diff --git a/internal/backend/util/defaults_test.go b/internal/backend/util/defaults_test.go new file mode 100644 index 000000000..1dd79208f --- /dev/null +++ b/internal/backend/util/defaults_test.go @@ -0,0 +1,64 @@ +package util_test + +import ( + "context" + "io" + "testing" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/util" + "github.com/restic/restic/internal/errors" + + rtest "github.com/restic/restic/internal/test" +) + +type mockReader struct { + closed bool +} + +func (rd *mockReader) Read(_ []byte) (n int, err error) { + return 0, nil +} +func (rd *mockReader) Close() error { + rd.closed = true + return nil +} + +func TestDefaultLoad(t *testing.T) { + + h := backend.Handle{Name: "id", Type: backend.PackFile} + rd := &mockReader{} + + // happy case, assert correct parameters are passed around and content stream is closed + err := util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) { + rtest.Equals(t, h, ih) + rtest.Equals(t, int(10), length) + rtest.Equals(t, int64(11), offset) + + return rd, nil + }, func(ird io.Reader) error { + rtest.Equals(t, rd, ird) + return nil + }) + rtest.OK(t, err) + rtest.Equals(t, true, rd.closed) + + // unhappy case, assert producer errors are handled correctly + err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) { + return nil, errors.Errorf("producer error") + }, func(ird io.Reader) error { + t.Fatalf("unexpected consumer invocation") + return nil + }) + rtest.Equals(t, "producer error", err.Error()) + + // unhappy case, assert consumer errors are handled correctly + rd = &mockReader{} + err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) { + return rd, nil + }, func(ird io.Reader) error { + return errors.Errorf("consumer error") + }) + rtest.Equals(t, true, rd.closed) + rtest.Equals(t, "consumer error", err.Error()) +} diff --git a/internal/backend/errdot_119.go b/internal/backend/util/errdot_119.go similarity index 97% rename from internal/backend/errdot_119.go rename to internal/backend/util/errdot_119.go index 3676a099d..e20ed47b7 100644 --- a/internal/backend/errdot_119.go +++ b/internal/backend/util/errdot_119.go @@ -8,7 +8,7 @@ // Once the minimum Go version restic supports is 1.19, remove this file and // replace any calls to it with the corresponding code as per below. -package backend +package util import ( "errors" diff --git a/internal/backend/errdot_old.go b/internal/backend/util/errdot_old.go similarity index 95% rename from internal/backend/errdot_old.go rename to internal/backend/util/errdot_old.go index 92a58ad25..4f7a0b40b 100644 --- a/internal/backend/errdot_old.go +++ b/internal/backend/util/errdot_old.go @@ -6,7 +6,7 @@ // Once the minimum Go version restic supports is 1.19, remove this file // and perform the actions listed in errdot_119.go. -package backend +package util func IsErrDot(err error) bool { return false diff --git a/internal/backend/foreground.go b/internal/backend/util/foreground.go similarity index 97% rename from internal/backend/foreground.go rename to internal/backend/util/foreground.go index 7291dc8d6..35cbada1a 100644 --- a/internal/backend/foreground.go +++ b/internal/backend/util/foreground.go @@ -1,4 +1,4 @@ -package backend +package util import ( "os" diff --git a/internal/backend/foreground_sysv.go b/internal/backend/util/foreground_sysv.go similarity index 85% rename from internal/backend/foreground_sysv.go rename to internal/backend/util/foreground_sysv.go index 0e88a57a1..ec06aa677 100644 --- a/internal/backend/foreground_sysv.go +++ b/internal/backend/util/foreground_sysv.go @@ -1,7 +1,7 @@ //go:build aix || solaris // +build aix solaris -package backend +package util import ( "os/exec" @@ -11,7 +11,7 @@ import ( ) func startForeground(cmd *exec.Cmd) (bg func() error, err error) { - // run the command in it's own process group so that SIGINT + // run the command in its own process group so that SIGINT // is not sent to it. cmd.SysProcAttr = &syscall.SysProcAttr{ Setpgid: true, diff --git a/internal/backend/foreground_test.go b/internal/backend/util/foreground_test.go similarity index 85% rename from internal/backend/foreground_test.go rename to internal/backend/util/foreground_test.go index 4f701122d..c26861a6c 100644 --- a/internal/backend/foreground_test.go +++ b/internal/backend/util/foreground_test.go @@ -1,7 +1,7 @@ //go:build !windows // +build !windows -package backend_test +package util_test import ( "bufio" @@ -10,7 +10,7 @@ import ( "strings" "testing" - "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/util" rtest "github.com/restic/restic/internal/test" ) @@ -22,7 +22,7 @@ func TestForeground(t *testing.T) { stdout, err := cmd.StdoutPipe() rtest.OK(t, err) - bg, err := backend.StartForeground(cmd) + bg, err := util.StartForeground(cmd) rtest.OK(t, err) defer func() { rtest.OK(t, cmd.Wait()) diff --git a/internal/backend/foreground_unix.go b/internal/backend/util/foreground_unix.go similarity index 98% rename from internal/backend/foreground_unix.go rename to internal/backend/util/foreground_unix.go index fcc0dfe78..082b7f59b 100644 --- a/internal/backend/foreground_unix.go +++ b/internal/backend/util/foreground_unix.go @@ -1,7 +1,7 @@ //go:build !aix && !solaris && !windows // +build !aix,!solaris,!windows -package backend +package util import ( "os" diff --git a/internal/backend/foreground_windows.go b/internal/backend/util/foreground_windows.go similarity index 96% rename from internal/backend/foreground_windows.go rename to internal/backend/util/foreground_windows.go index 54883c30f..f9b753c35 100644 --- a/internal/backend/foreground_windows.go +++ b/internal/backend/util/foreground_windows.go @@ -1,4 +1,4 @@ -package backend +package util import ( "os/exec" diff --git a/internal/backend/paths.go b/internal/backend/util/paths.go similarity index 97% rename from internal/backend/paths.go rename to internal/backend/util/paths.go index 7e511be9c..206fbb56d 100644 --- a/internal/backend/paths.go +++ b/internal/backend/util/paths.go @@ -1,4 +1,4 @@ -package backend +package util import "os" diff --git a/internal/backend/utils.go b/internal/backend/utils.go index cd6614f34..161608295 100644 --- a/internal/backend/utils.go +++ b/internal/backend/utils.go @@ -3,18 +3,36 @@ package backend import ( "bytes" "context" + "encoding/hex" "fmt" "io" + "github.com/minio/sha256-simd" + "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" ) +func verifyContentMatchesName(s string, data []byte) (bool, error) { + if len(s) != hex.EncodedLen(sha256.Size) { + return false, fmt.Errorf("invalid length for ID: %q", s) + } + + b, err := hex.DecodeString(s) + if err != nil { + return false, fmt.Errorf("invalid ID: %s", err) + } + var id [sha256.Size]byte + copy(id[:], b) + + hashed := sha256.Sum256(data) + return id == hashed, nil +} + // LoadAll reads all data stored in the backend for the handle into the given // buffer, which is truncated. If the buffer is not large enough or nil, a new // one is allocated. -func LoadAll(ctx context.Context, buf []byte, be restic.Backend, h restic.Handle) ([]byte, error) { +func LoadAll(ctx context.Context, buf []byte, be Backend, h Handle) ([]byte, error) { retriedInvalidData := false err := be.Load(ctx, h, 0, 0, func(rd io.Reader) error { // make sure this is idempotent, in case an error occurs this function may be called multiple times! @@ -28,9 +46,8 @@ func LoadAll(ctx context.Context, buf []byte, be restic.Backend, h restic.Handle // retry loading damaged data only once. If a file fails to download correctly // the second time, then it is likely corrupted at the backend. Return the data // to the caller in that case to let it decide what to do with the data. - if !retriedInvalidData && h.Type != restic.ConfigFile { - id, err := restic.ParseID(h.Name) - if err == nil && !restic.Hash(buf).Equal(id) { + if !retriedInvalidData && h.Type != ConfigFile { + if matches, err := verifyContentMatchesName(h.Name, buf); err == nil && !matches { debug.Log("retry loading broken blob %v", h) retriedInvalidData = true return errors.Errorf("loadAll(%v): invalid data returned", h) @@ -57,86 +74,3 @@ type LimitedReadCloser struct { func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser { return &LimitedReadCloser{Closer: r, LimitedReader: io.LimitedReader{R: r, N: n}} } - -// DefaultLoad implements Backend.Load using lower-level openReader func -func DefaultLoad(ctx context.Context, h restic.Handle, length int, offset int64, - openReader func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error), - fn func(rd io.Reader) error) error { - - rd, err := openReader(ctx, h, length, offset) - if err != nil { - return err - } - err = fn(rd) - if err != nil { - _ = rd.Close() // ignore secondary errors closing the reader - return err - } - return rd.Close() -} - -// DefaultDelete removes all restic keys in the bucket. It will not remove the bucket itself. -func DefaultDelete(ctx context.Context, be restic.Backend) error { - alltypes := []restic.FileType{ - restic.PackFile, - restic.KeyFile, - restic.LockFile, - restic.SnapshotFile, - restic.IndexFile} - - for _, t := range alltypes { - err := be.List(ctx, t, func(fi restic.FileInfo) error { - return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) - }) - if err != nil { - return nil - } - } - err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) - if err != nil && be.IsNotExist(err) { - err = nil - } - - return err -} - -type memorizedLister struct { - fileInfos []restic.FileInfo - tpe restic.FileType -} - -func (m *memorizedLister) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - if t != m.tpe { - return fmt.Errorf("filetype mismatch, expected %s got %s", m.tpe, t) - } - for _, fi := range m.fileInfos { - if ctx.Err() != nil { - break - } - err := fn(fi) - if err != nil { - return err - } - } - return ctx.Err() -} - -func MemorizeList(ctx context.Context, be restic.Lister, t restic.FileType) (restic.Lister, error) { - if _, ok := be.(*memorizedLister); ok { - return be, nil - } - - var fileInfos []restic.FileInfo - err := be.List(ctx, t, func(fi restic.FileInfo) error { - fileInfos = append(fileInfos, fi) - return nil - }) - if err != nil { - return nil, err - } - - return &memorizedLister{ - fileInfos: fileInfos, - tpe: t, - }, nil -} diff --git a/internal/backend/utils_test.go b/internal/backend/utils_test.go index 8392bfa8f..ad9540e54 100644 --- a/internal/backend/utils_test.go +++ b/internal/backend/utils_test.go @@ -3,7 +3,6 @@ package backend_test import ( "bytes" "context" - "fmt" "io" "math/rand" "testing" @@ -11,7 +10,6 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/backend/mock" - "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -27,11 +25,11 @@ func TestLoadAll(t *testing.T) { data := rtest.Random(23+i, rand.Intn(MiB)+500*KiB) id := restic.Hash(data) - h := restic.Handle{Name: id.String(), Type: restic.PackFile} - err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher())) + h := backend.Handle{Name: id.String(), Type: backend.PackFile} + err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher())) rtest.OK(t, err) - buf, err := backend.LoadAll(context.TODO(), buf, b, restic.Handle{Type: restic.PackFile, Name: id.String()}) + buf, err := backend.LoadAll(context.TODO(), buf, b, backend.Handle{Type: backend.PackFile, Name: id.String()}) rtest.OK(t, err) if len(buf) != len(data) { @@ -46,10 +44,10 @@ func TestLoadAll(t *testing.T) { } } -func save(t testing.TB, be restic.Backend, buf []byte) restic.Handle { +func save(t testing.TB, be backend.Backend, buf []byte) backend.Handle { id := restic.Hash(buf) - h := restic.Handle{Name: id.String(), Type: restic.PackFile} - err := be.Save(context.TODO(), h, restic.NewByteReader(buf, be.Hasher())) + h := backend.Handle{Name: id.String(), Type: backend.PackFile} + err := be.Save(context.TODO(), h, backend.NewByteReader(buf, be.Hasher())) if err != nil { t.Fatal(err) } @@ -57,10 +55,10 @@ func save(t testing.TB, be restic.Backend, buf []byte) restic.Handle { } type quickRetryBackend struct { - restic.Backend + backend.Backend } -func (be *quickRetryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *quickRetryBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { err := be.Backend.Load(ctx, h, length, offset, fn) if err != nil { // retry @@ -77,19 +75,19 @@ func TestLoadAllBroken(t *testing.T) { // damage buffer data[0] ^= 0xff - b.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + b.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(data)), nil } // must fail on first try - _, err := backend.LoadAll(context.TODO(), nil, b, restic.Handle{Type: restic.PackFile, Name: id.String()}) + _, err := backend.LoadAll(context.TODO(), nil, b, backend.Handle{Type: backend.PackFile, Name: id.String()}) if err == nil { t.Fatalf("missing expected error") } // must return the broken data after a retry be := &quickRetryBackend{Backend: b} - buf, err := backend.LoadAll(context.TODO(), nil, be, restic.Handle{Type: restic.PackFile, Name: id.String()}) + buf, err := backend.LoadAll(context.TODO(), nil, be, backend.Handle{Type: backend.PackFile, Name: id.String()}) rtest.OK(t, err) if !bytes.Equal(buf, data) { @@ -105,7 +103,7 @@ func TestLoadAllAppend(t *testing.T) { h2 := save(t, b, randomData) var tests = []struct { - handle restic.Handle + handle backend.Handle buf []byte want []byte }{ @@ -149,98 +147,3 @@ func TestLoadAllAppend(t *testing.T) { }) } } - -type mockReader struct { - closed bool -} - -func (rd *mockReader) Read(_ []byte) (n int, err error) { - return 0, nil -} -func (rd *mockReader) Close() error { - rd.closed = true - return nil -} - -func TestDefaultLoad(t *testing.T) { - - h := restic.Handle{Name: "id", Type: restic.PackFile} - rd := &mockReader{} - - // happy case, assert correct parameters are passed around and content stream is closed - err := backend.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) { - rtest.Equals(t, h, ih) - rtest.Equals(t, int(10), length) - rtest.Equals(t, int64(11), offset) - - return rd, nil - }, func(ird io.Reader) error { - rtest.Equals(t, rd, ird) - return nil - }) - rtest.OK(t, err) - rtest.Equals(t, true, rd.closed) - - // unhappy case, assert producer errors are handled correctly - err = backend.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) { - return nil, errors.Errorf("producer error") - }, func(ird io.Reader) error { - t.Fatalf("unexpected consumer invocation") - return nil - }) - rtest.Equals(t, "producer error", err.Error()) - - // unhappy case, assert consumer errors are handled correctly - rd = &mockReader{} - err = backend.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) { - return rd, nil - }, func(ird io.Reader) error { - return errors.Errorf("consumer error") - }) - rtest.Equals(t, true, rd.closed) - rtest.Equals(t, "consumer error", err.Error()) -} - -func TestMemoizeList(t *testing.T) { - // setup backend to serve as data source for memoized list - be := mock.NewBackend() - files := []restic.FileInfo{ - {Size: 42, Name: restic.NewRandomID().String()}, - {Size: 45, Name: restic.NewRandomID().String()}, - } - be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - for _, fi := range files { - if err := fn(fi); err != nil { - return err - } - } - return nil - } - - mem, err := backend.MemorizeList(context.TODO(), be, restic.SnapshotFile) - rtest.OK(t, err) - - err = mem.List(context.TODO(), restic.IndexFile, func(fi restic.FileInfo) error { - t.Fatal("file type mismatch") - return nil // the memoized lister must return an error by itself - }) - rtest.Assert(t, err != nil, "missing error on file typ mismatch") - - var memFiles []restic.FileInfo - err = mem.List(context.TODO(), restic.SnapshotFile, func(fi restic.FileInfo) error { - memFiles = append(memFiles, fi) - return nil - }) - rtest.OK(t, err) - rtest.Equals(t, files, memFiles) -} - -func TestMemoizeListError(t *testing.T) { - // setup backend to serve as data source for memoized list - be := mock.NewBackend() - be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - return fmt.Errorf("list error") - } - _, err := backend.MemorizeList(context.TODO(), be, restic.SnapshotFile) - rtest.Assert(t, err != nil, "missing error on list error") -} diff --git a/internal/cache/backend.go b/internal/cache/backend.go index 311b099ee..5cbdb5444 100644 --- a/internal/cache/backend.go +++ b/internal/cache/backend.go @@ -5,35 +5,35 @@ import ( "io" "sync" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/restic" ) // Backend wraps a restic.Backend and adds a cache. type Backend struct { - restic.Backend + backend.Backend *Cache // inProgress contains the handle for all files that are currently // downloaded. The channel in the value is closed as soon as the download // is finished. inProgressMutex sync.Mutex - inProgress map[restic.Handle]chan struct{} + inProgress map[backend.Handle]chan struct{} } -// ensure Backend implements restic.Backend -var _ restic.Backend = &Backend{} +// ensure Backend implements backend.Backend +var _ backend.Backend = &Backend{} -func newBackend(be restic.Backend, c *Cache) *Backend { +func newBackend(be backend.Backend, c *Cache) *Backend { return &Backend{ Backend: be, Cache: c, - inProgress: make(map[restic.Handle]chan struct{}), + inProgress: make(map[backend.Handle]chan struct{}), } } // Remove deletes a file from the backend and the cache if it has been cached. -func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (b *Backend) Remove(ctx context.Context, h backend.Handle) error { debug.Log("cache Remove(%v)", h) err := b.Backend.Remove(ctx, h) if err != nil { @@ -43,18 +43,18 @@ func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { return b.Cache.remove(h) } -func autoCacheTypes(h restic.Handle) bool { +func autoCacheTypes(h backend.Handle) bool { switch h.Type { - case restic.IndexFile, restic.SnapshotFile: + case backend.IndexFile, backend.SnapshotFile: return true - case restic.PackFile: - return h.ContainedBlobType == restic.TreeBlob + case backend.PackFile: + return h.IsMetadata } return false } // Save stores a new file in the backend and the cache. -func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { if !autoCacheTypes(h) { return b.Backend.Save(ctx, h, rd) } @@ -89,7 +89,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea return nil } -func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error { +func (b *Backend) cacheFile(ctx context.Context, h backend.Handle) error { finish := make(chan struct{}) b.inProgressMutex.Lock() @@ -133,7 +133,7 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error { } // loadFromCache will try to load the file from the cache. -func (b *Backend) loadFromCache(h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) { +func (b *Backend) loadFromCache(h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) { rd, err := b.Cache.load(h, length, offset) if err != nil { return false, err @@ -148,7 +148,7 @@ func (b *Backend) loadFromCache(h restic.Handle, length int, offset int64, consu } // Load loads a file from the cache or the backend. -func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { +func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { b.inProgressMutex.Lock() waitForFinish, inProgress := b.inProgress[h] b.inProgressMutex.Unlock() @@ -194,7 +194,7 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset // Stat tests whether the backend has a file. If it does not exist but still // exists in the cache, it is removed from the cache. -func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { debug.Log("cache Stat(%v)", h) fi, err := b.Backend.Stat(ctx, h) @@ -215,6 +215,6 @@ func (b *Backend) IsNotExist(err error) bool { return b.Backend.IsNotExist(err) } -func (b *Backend) Unwrap() restic.Backend { +func (b *Backend) Unwrap() backend.Backend { return b.Backend } diff --git a/internal/cache/backend_test.go b/internal/cache/backend_test.go index 930d853b2..68fbb02b3 100644 --- a/internal/cache/backend_test.go +++ b/internal/cache/backend_test.go @@ -16,7 +16,7 @@ import ( "github.com/restic/restic/internal/test" ) -func loadAndCompare(t testing.TB, be restic.Backend, h restic.Handle, data []byte) { +func loadAndCompare(t testing.TB, be backend.Backend, h backend.Handle, data []byte) { buf, err := backend.LoadAll(context.TODO(), nil, be, h) if err != nil { t.Fatal(err) @@ -31,25 +31,25 @@ func loadAndCompare(t testing.TB, be restic.Backend, h restic.Handle, data []byt } } -func save(t testing.TB, be restic.Backend, h restic.Handle, data []byte) { - err := be.Save(context.TODO(), h, restic.NewByteReader(data, be.Hasher())) +func save(t testing.TB, be backend.Backend, h backend.Handle, data []byte) { + err := be.Save(context.TODO(), h, backend.NewByteReader(data, be.Hasher())) if err != nil { t.Fatal(err) } } -func remove(t testing.TB, be restic.Backend, h restic.Handle) { +func remove(t testing.TB, be backend.Backend, h backend.Handle) { err := be.Remove(context.TODO(), h) if err != nil { t.Fatal(err) } } -func randomData(n int) (restic.Handle, []byte) { +func randomData(n int) (backend.Handle, []byte) { data := test.Random(rand.Int(), n) id := restic.Hash(data) - h := restic.Handle{ - Type: restic.IndexFile, + h := backend.Handle{ + Type: backend.IndexFile, Name: id.String(), } return h, data @@ -114,11 +114,11 @@ func TestBackend(t *testing.T) { } type loadErrorBackend struct { - restic.Backend + backend.Backend loadError error } -func (be loadErrorBackend) Load(_ context.Context, _ restic.Handle, _ int, _ int64, _ func(rd io.Reader) error) error { +func (be loadErrorBackend) Load(_ context.Context, _ backend.Handle, _ int, _ int64, _ func(rd io.Reader) error) error { time.Sleep(10 * time.Millisecond) return be.loadError } @@ -137,7 +137,7 @@ func TestErrorBackend(t *testing.T) { loadError: testErr, } - loadTest := func(wg *sync.WaitGroup, be restic.Backend) { + loadTest := func(wg *sync.WaitGroup, be backend.Backend) { defer wg.Done() buf, err := backend.LoadAll(context.TODO(), nil, be, h) diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 5b3601741..19b3182df 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -9,6 +9,7 @@ import ( "time" "github.com/pkg/errors" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" @@ -234,7 +235,7 @@ func IsOld(t time.Time, maxAge time.Duration) bool { } // Wrap returns a backend with a cache. -func (c *Cache) Wrap(be restic.Backend) restic.Backend { +func (c *Cache) Wrap(be backend.Backend) backend.Backend { return newBackend(be, c) } diff --git a/internal/cache/file.go b/internal/cache/file.go index c315be19f..48a38c1d3 100644 --- a/internal/cache/file.go +++ b/internal/cache/file.go @@ -14,7 +14,7 @@ import ( "github.com/restic/restic/internal/restic" ) -func (c *Cache) filename(h restic.Handle) string { +func (c *Cache) filename(h backend.Handle) string { if len(h.Name) < 2 { panic("Name is empty or too short") } @@ -22,7 +22,7 @@ func (c *Cache) filename(h restic.Handle) string { return filepath.Join(c.path, cacheLayoutPaths[h.Type], subdir, h.Name) } -func (c *Cache) canBeCached(t restic.FileType) bool { +func (c *Cache) canBeCached(t backend.FileType) bool { if c == nil { return false } @@ -34,7 +34,7 @@ func (c *Cache) canBeCached(t restic.FileType) bool { // Load returns a reader that yields the contents of the file with the // given handle. rd must be closed after use. If an error is returned, the // ReadCloser is nil. -func (c *Cache) load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (c *Cache) load(h backend.Handle, length int, offset int64) (io.ReadCloser, error) { debug.Log("Load(%v, %v, %v) from cache", h, length, offset) if !c.canBeCached(h.Type) { return nil, errors.New("cannot be cached") @@ -78,7 +78,7 @@ func (c *Cache) load(h restic.Handle, length int, offset int64) (io.ReadCloser, } // Save saves a file in the cache. -func (c *Cache) Save(h restic.Handle, rd io.Reader) error { +func (c *Cache) Save(h backend.Handle, rd io.Reader) error { debug.Log("Save to cache: %v", h) if rd == nil { return errors.New("Save() called with nil reader") @@ -139,7 +139,7 @@ func (c *Cache) Save(h restic.Handle, rd io.Reader) error { } // Remove deletes a file. When the file is not cache, no error is returned. -func (c *Cache) remove(h restic.Handle) error { +func (c *Cache) remove(h backend.Handle) error { if !c.Has(h) { return nil } @@ -165,7 +165,7 @@ func (c *Cache) Clear(t restic.FileType, valid restic.IDSet) error { continue } - if err = fs.Remove(c.filename(restic.Handle{Type: t, Name: id.String()})); err != nil { + if err = fs.Remove(c.filename(backend.Handle{Type: t, Name: id.String()})); err != nil { return err } } @@ -207,7 +207,7 @@ func (c *Cache) list(t restic.FileType) (restic.IDSet, error) { } // Has returns true if the file is cached. -func (c *Cache) Has(h restic.Handle) bool { +func (c *Cache) Has(h backend.Handle) bool { if !c.canBeCached(h.Type) { return false } diff --git a/internal/cache/file_test.go b/internal/cache/file_test.go index e72133cd7..7935f9806 100644 --- a/internal/cache/file_test.go +++ b/internal/cache/file_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" @@ -18,12 +19,12 @@ import ( "golang.org/x/sync/errgroup" ) -func generateRandomFiles(t testing.TB, tpe restic.FileType, c *Cache) restic.IDSet { +func generateRandomFiles(t testing.TB, tpe backend.FileType, c *Cache) restic.IDSet { ids := restic.NewIDSet() for i := 0; i < rand.Intn(15)+10; i++ { buf := test.Random(rand.Int(), 1<<19) id := restic.Hash(buf) - h := restic.Handle{Type: tpe, Name: id.String()} + h := backend.Handle{Type: tpe, Name: id.String()} if c.Has(h) { t.Errorf("index %v present before save", id) @@ -46,7 +47,7 @@ func randomID(s restic.IDSet) restic.ID { panic("set is empty") } -func load(t testing.TB, c *Cache, h restic.Handle) []byte { +func load(t testing.TB, c *Cache, h backend.Handle) []byte { rd, err := c.load(h, 0, 0) if err != nil { t.Fatal(err) @@ -101,7 +102,7 @@ func TestFiles(t *testing.T) { ids := generateRandomFiles(t, tpe, c) id := randomID(ids) - h := restic.Handle{Type: tpe, Name: id.String()} + h := backend.Handle{Type: tpe, Name: id.String()} id2 := restic.Hash(load(t, c, h)) if !id.Equal(id2) { @@ -146,7 +147,7 @@ func TestFileLoad(t *testing.T) { data := test.Random(rand.Int(), 5234142) id := restic.ID{} copy(id[:], data) - h := restic.Handle{ + h := backend.Handle{ Type: restic.PackFile, Name: id.String(), } @@ -230,7 +231,7 @@ func TestFileSaveConcurrent(t *testing.T) { ) rand.Read(id[:]) - h := restic.Handle{ + h := backend.Handle{ Type: restic.PackFile, Name: id.String(), } @@ -275,7 +276,7 @@ func TestFileSaveAfterDamage(t *testing.T) { // save a few bytes of data in the cache data := test.Random(123456789, 42) id := restic.Hash(data) - h := restic.Handle{ + h := backend.Handle{ Type: restic.PackFile, Name: id.String(), } diff --git a/internal/checker/checker.go b/internal/checker/checker.go index b2512969e..3bc0fac87 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -90,9 +90,19 @@ func (err *ErrOldIndexFormat) Error() string { return fmt.Sprintf("index %v has old format", err.ID) } +// ErrPackData is returned if errors are discovered while verifying a packfile +type ErrPackData struct { + PackID restic.ID + errs []error +} + +func (e *ErrPackData) Error() string { + return fmt.Sprintf("pack %v contains %v errors: %v", e.PackID, len(e.errs), e.errs) +} + func (c *Checker) LoadSnapshots(ctx context.Context) error { var err error - c.snapshots, err = backend.MemorizeList(ctx, c.repo.Backend(), restic.SnapshotFile) + c.snapshots, err = restic.MemorizeList(ctx, c.repo, restic.SnapshotFile) return err } @@ -113,13 +123,36 @@ func computePackTypes(ctx context.Context, idx restic.MasterIndex) map[restic.ID } // LoadIndex loads all index files. -func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) { +func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []error, errs []error) { debug.Log("Start") + indexList, err := restic.MemorizeList(ctx, c.repo, restic.IndexFile) + if err != nil { + // abort if an error occurs while listing the indexes + return hints, append(errs, err) + } + + if p != nil { + var numIndexFiles uint64 + err := indexList.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error { + numIndexFiles++ + return nil + }) + if err != nil { + return hints, append(errs, err) + } + p.SetMax(numIndexFiles) + defer p.Done() + } + packToIndex := make(map[restic.ID]restic.IDSet) - err := index.ForAllIndexes(ctx, c.repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { + err = index.ForAllIndexes(ctx, indexList, c.repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { debug.Log("process index %v, err %v", id, err) + if p != nil { + p.Add(1) + } + if oldFormat { debug.Log("index %v has old format", id) hints = append(hints, &ErrOldIndexFormat{id}) @@ -206,7 +239,7 @@ func IsOrphanedPack(err error) bool { return errors.As(err, &e) && e.Orphaned } -func isS3Legacy(b restic.Backend) bool { +func isS3Legacy(b backend.Backend) bool { // unwrap cache if be, ok := b.(*cache.Backend); ok { b = be.Backend @@ -409,7 +442,7 @@ func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) { } // Note that we do not use the blob size. The "obvious" check // whether the sum of the blob sizes matches the file size - // unfortunately fails in some cases that are not resolveable + // unfortunately fails in some cases that are not resolvable // by users, so we omit this check, see #1887 _, found := c.repo.LookupBlobSize(blobID, restic.DataBlob) @@ -524,7 +557,7 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []r // calculate hash on-the-fly while reading the pack and capture pack header var hash restic.ID var hdrBuf []byte - hashingLoader := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + hashingLoader := func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { return r.Backend().Load(ctx, h, int(size), 0, func(rd io.Reader) error { hrd := hashing.NewReader(rd, sha256.New()) bufRd.Reset(hrd) @@ -606,7 +639,7 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []r } if len(errs) > 0 { - return errors.Errorf("pack %v contains %v errors: %v", id, len(errs), errs) + return &ErrPackData{PackID: id, errs: errs} } return nil diff --git a/internal/checker/checker_test.go b/internal/checker/checker_test.go index 6405ecfbd..cca5a582c 100644 --- a/internal/checker/checker_test.go +++ b/internal/checker/checker_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/restic/restic/internal/archiver" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/hashing" @@ -77,7 +78,7 @@ func TestCheckRepo(t *testing.T) { repo := repository.TestOpenLocal(t, repodir) chkr := checker.New(repo, false) - hints, errs := chkr.LoadIndex(context.TODO()) + hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } @@ -96,14 +97,14 @@ func TestMissingPack(t *testing.T) { repo := repository.TestOpenLocal(t, repodir) - packHandle := restic.Handle{ + packHandle := backend.Handle{ Type: restic.PackFile, Name: "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6", } test.OK(t, repo.Backend().Remove(context.TODO(), packHandle)) chkr := checker.New(repo, false) - hints, errs := chkr.LoadIndex(context.TODO()) + hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } @@ -129,14 +130,14 @@ func TestUnreferencedPack(t *testing.T) { // index 3f1a only references pack 60e0 packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e" - indexHandle := restic.Handle{ + indexHandle := backend.Handle{ Type: restic.IndexFile, Name: "3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44", } test.OK(t, repo.Backend().Remove(context.TODO(), indexHandle)) chkr := checker.New(repo, false) - hints, errs := chkr.LoadIndex(context.TODO()) + hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } @@ -160,7 +161,7 @@ func TestUnreferencedBlobs(t *testing.T) { repo := repository.TestOpenLocal(t, repodir) - snapshotHandle := restic.Handle{ + snapshotHandle := backend.Handle{ Type: restic.SnapshotFile, Name: "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02", } @@ -178,7 +179,7 @@ func TestUnreferencedBlobs(t *testing.T) { sort.Sort(unusedBlobsBySnapshot) chkr := checker.New(repo, true) - hints, errs := chkr.LoadIndex(context.TODO()) + hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } @@ -202,7 +203,7 @@ func TestModifiedIndex(t *testing.T) { done := make(chan struct{}) defer close(done) - h := restic.Handle{ + h := backend.Handle{ Type: restic.IndexFile, Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd", } @@ -238,7 +239,7 @@ func TestModifiedIndex(t *testing.T) { // save the index again with a modified name so that the hash doesn't match // the content any more - h2 := restic.Handle{ + h2 := backend.Handle{ Type: restic.IndexFile, Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd", } @@ -247,7 +248,7 @@ func TestModifiedIndex(t *testing.T) { if hw != nil { hash = hw.Sum(nil) } - rd, err := restic.NewFileReader(tmpfile, hash) + rd, err := backend.NewFileReader(tmpfile, hash) if err != nil { t.Fatal(err) } @@ -258,7 +259,7 @@ func TestModifiedIndex(t *testing.T) { } chkr := checker.New(repo, false) - hints, errs := chkr.LoadIndex(context.TODO()) + hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(errs) == 0 { t.Fatalf("expected errors not found") } @@ -279,7 +280,7 @@ func TestDuplicatePacksInIndex(t *testing.T) { repo := repository.TestOpenLocal(t, repodir) chkr := checker.New(repo, false) - hints, errs := chkr.LoadIndex(context.TODO()) + hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(hints) == 0 { t.Fatalf("did not get expected checker hints for duplicate packs in indexes") } @@ -304,11 +305,11 @@ func TestDuplicatePacksInIndex(t *testing.T) { // errorBackend randomly modifies data after reading. type errorBackend struct { - restic.Backend + backend.Backend ProduceErrors bool } -func (b errorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { +func (b errorBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error { if b.ProduceErrors { return consumer(errorReadCloser{rd}) @@ -347,7 +348,7 @@ func TestCheckerModifiedData(t *testing.T) { chkr := checker.New(checkRepo, false) - hints, errs := chkr.LoadIndex(context.TODO()) + hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } @@ -408,7 +409,7 @@ func TestCheckerNoDuplicateTreeDecodes(t *testing.T) { } chkr := checker.New(checkRepo, false) - hints, errs := chkr.LoadIndex(context.TODO()) + hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } @@ -524,7 +525,7 @@ func TestCheckerBlobTypeConfusion(t *testing.T) { delayRepo.Unblock() }() - hints, errs := chkr.LoadIndex(ctx) + hints, errs := chkr.LoadIndex(ctx, nil) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } @@ -553,7 +554,7 @@ func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository, fun repo := repository.TestOpenLocal(t, repodir) chkr := checker.New(repo, false) - hints, errs := chkr.LoadIndex(context.TODO()) + hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(errs) > 0 { defer cleanup() t.Fatalf("expected no errors, got %v: %v", len(errs), errs) diff --git a/internal/checker/testing.go b/internal/checker/testing.go index 0668406d8..fe1679393 100644 --- a/internal/checker/testing.go +++ b/internal/checker/testing.go @@ -11,7 +11,7 @@ import ( func TestCheckRepo(t testing.TB, repo restic.Repository) { chkr := New(repo, true) - hints, errs := chkr.LoadIndex(context.TODO()) + hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(errs) != 0 { t.Fatalf("errors loading index: %v", errs) } diff --git a/internal/dump/acl.go b/internal/dump/acl.go index 48ce28a4c..2735ffa46 100644 --- a/internal/dump/acl.go +++ b/internal/dump/acl.go @@ -1,131 +1,88 @@ package dump -// Adapted from https://github.com/maxymania/go-system/blob/master/posix_acl/posix_acl.go - import ( - "bytes" "encoding/binary" - "fmt" + "errors" + "strconv" ) const ( - aclUserOwner = 0x0001 - aclUser = 0x0002 - aclGroupOwner = 0x0004 - aclGroup = 0x0008 - aclMask = 0x0010 - aclOthers = 0x0020 + // Permissions + aclPermRead = 0x4 + aclPermWrite = 0x2 + aclPermExecute = 0x1 + + // Tags + aclTagUserObj = 0x01 // Owner. + aclTagUser = 0x02 + aclTagGroupObj = 0x04 // Owning group. + aclTagGroup = 0x08 + aclTagMask = 0x10 + aclTagOther = 0x20 ) -type aclSID uint64 - -type aclElem struct { - Tag uint16 - Perm uint16 - ID uint32 -} - -type acl struct { - Version uint32 - List []aclElement -} - -type aclElement struct { - aclSID - Perm uint16 -} - -func (a aclSID) getType() int { - return int(a >> 32) -} -func (a aclSID) getID() uint32 { - return uint32(a & 0xffffffff) -} -func (a aclSID) String() string { - switch a >> 32 { - case aclUserOwner: - return "user::" - case aclUser: - return fmt.Sprintf("user:%v:", a.getID()) - case aclGroupOwner: - return "group::" - case aclGroup: - return fmt.Sprintf("group:%v:", a.getID()) - case aclMask: - return "mask::" - case aclOthers: - return "other::" +// formatLinuxACL converts a Linux ACL from its binary format to the POSIX.1e +// long text format. +// +// User and group IDs are printed in decimal, because we may be dumping +// a snapshot from a different machine. +// +// https://man7.org/linux/man-pages/man5/acl.5.html +// https://savannah.nongnu.org/projects/acl +// https://simson.net/ref/1997/posix_1003.1e-990310.pdf +func formatLinuxACL(acl []byte) (string, error) { + if len(acl)-4 < 0 || (len(acl)-4)%8 != 0 { + return "", errors.New("wrong length") } - return "?:" -} + version := binary.LittleEndian.Uint32(acl) + if version != 2 { + return "", errors.New("unsupported ACL format version") + } + acl = acl[4:] -func (a aclElement) String() string { - str := "" - if (a.Perm & 4) != 0 { - str += "r" - } else { - str += "-" - } - if (a.Perm & 2) != 0 { - str += "w" - } else { - str += "-" - } - if (a.Perm & 1) != 0 { - str += "x" - } else { - str += "-" - } - return fmt.Sprintf("%v%v", a.aclSID, str) -} + text := make([]byte, 0, 2*len(acl)) -func (a *acl) decode(xattr []byte) { - var elem aclElement - ae := new(aclElem) - nr := bytes.NewReader(xattr) - e := binary.Read(nr, binary.LittleEndian, &a.Version) - if e != nil { - a.Version = 0 - return - } - if len(a.List) > 0 { - a.List = a.List[:0] - } - for binary.Read(nr, binary.LittleEndian, ae) == nil { - elem.aclSID = (aclSID(ae.Tag) << 32) | aclSID(ae.ID) - elem.Perm = ae.Perm - a.List = append(a.List, elem) - } -} + for ; len(acl) >= 8; acl = acl[8:] { + tag := binary.LittleEndian.Uint16(acl) + perm := binary.LittleEndian.Uint16(acl[2:]) + id := binary.LittleEndian.Uint32(acl[4:]) -func (a *acl) encode() []byte { - buf := new(bytes.Buffer) - ae := new(aclElem) - - err := binary.Write(buf, binary.LittleEndian, &a.Version) - // write to a bytes.Buffer always returns a nil error - if err != nil { - panic(err) - } - - for _, elem := range a.List { - ae.Tag = uint16(elem.getType()) - ae.Perm = elem.Perm - ae.ID = elem.getID() - - err := binary.Write(buf, binary.LittleEndian, ae) - // write to a bytes.Buffer always returns a nil error - if err != nil { - panic(err) + switch tag { + case aclTagUserObj: + text = append(text, "user:"...) + case aclTagUser: + text = append(text, "user:"...) + text = strconv.AppendUint(text, uint64(id), 10) + case aclTagGroupObj: + text = append(text, "group:"...) + case aclTagGroup: + text = append(text, "group:"...) + text = strconv.AppendUint(text, uint64(id), 10) + case aclTagMask: + text = append(text, "mask:"...) + case aclTagOther: + text = append(text, "other:"...) + default: + return "", errors.New("unknown tag") } + text = append(text, ':') + text = append(text, aclPermText(perm)...) + text = append(text, '\n') } - return buf.Bytes() + + return string(text), nil } -func (a *acl) String() string { - var finalacl string - for _, acl := range a.List { - finalacl += acl.String() + "\n" +func aclPermText(p uint16) []byte { + s := []byte("---") + if p&aclPermRead != 0 { + s[0] = 'r' } - return finalacl + if p&aclPermWrite != 0 { + s[1] = 'w' + } + if p&aclPermExecute != 0 { + s[2] = 'x' + } + return s } diff --git a/internal/dump/acl_test.go b/internal/dump/acl_test.go index bef11ad14..658850147 100644 --- a/internal/dump/acl_test.go +++ b/internal/dump/acl_test.go @@ -1,114 +1,46 @@ package dump import ( - "reflect" "testing" + + rtest "github.com/restic/restic/internal/test" ) -func Test_acl_decode(t *testing.T) { - type args struct { - xattr []byte - } - tests := []struct { - name string - args args - want string +func TestFormatLinuxACL(t *testing.T) { + for _, c := range []struct { + in, out, err string }{ { - name: "decode string", - args: args{ - xattr: []byte{2, 0, 0, 0, 1, 0, 6, 0, 255, 255, 255, 255, 2, 0, 7, 0, 0, 0, 0, 0, 2, 0, 7, 0, 254, 255, 0, 0, 4, 0, 7, 0, 255, 255, 255, 255, 16, 0, 7, 0, 255, 255, 255, 255, 32, 0, 4, 0, 255, 255, 255, 255}, - }, - want: "user::rw-\nuser:0:rwx\nuser:65534:rwx\ngroup::rwx\nmask::rwx\nother::r--\n", + in: "\x02\x00\x00\x00\x01\x00\x06\x00\xff\xff\xff\xff\x02\x00" + + "\x04\x00\x03\x00\x00\x00\x02\x00\x04\x00\xe9\x03\x00\x00" + + "\x04\x00\x02\x00\xff\xff\xff\xff\b\x00\x01\x00'\x00\x00\x00" + + "\x10\x00\a\x00\xff\xff\xff\xff \x00\x04\x00\xff\xff\xff\xff", + out: "user::rw-\nuser:3:r--\nuser:1001:r--\ngroup::-w-\n" + + "group:39:--x\nmask::rwx\nother::r--\n", }, { - name: "decode group", - args: args{ - xattr: []byte{2, 0, 0, 0, 8, 0, 1, 0, 254, 255, 0, 0}, - }, - want: "group:65534:--x\n", + in: "\x02\x00\x00\x00\x00\x00\x06\x00\xff\xff\xff\xff\x02\x00" + + "\x04\x00\x03\x00\x00\x00\x02\x00\x04\x00\xe9\x03\x00\x00" + + "\x04\x00\x06\x00\xff\xff\xff\xff\b\x00\x05\x00'\x00\x00\x00" + + "\x10\x00\a\x00\xff\xff\xff\xff \x00\x04\x00\xff\xff\xff\xff", + err: "unknown tag", }, { - name: "decode fail", - args: args{ - xattr: []byte("abctest"), - }, - want: "", + in: "\x01\x00\x00\x00\x01\x00\x06\x00\xff\xff\xff\xff\x02\x00" + + "\x04\x00\x03\x00\x00\x00\x02\x00\x04\x00\xe9\x03\x00\x00" + + "\x04\x00\x06\x00\xff\xff\xff\xff\b\x00\x05\x00'\x00\x00\x00" + + "\x10\x00\a\x00\xff\xff\xff\xff \x00\x04\x00\xff\xff\xff\xff", + err: "unsupported ACL format version", }, - { - name: "decode empty fail", - args: args{ - xattr: []byte(""), - }, - want: "", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - a := &acl{} - a.decode(tt.args.xattr) - if tt.want != a.String() { - t.Errorf("acl.decode() = %v, want: %v", a.String(), tt.want) - } - a.decode(tt.args.xattr) - if tt.want != a.String() { - t.Errorf("second acl.decode() = %v, want: %v", a.String(), tt.want) - } - }) - } -} - -func Test_acl_encode(t *testing.T) { - tests := []struct { - name string - want []byte - args []aclElement - }{ - { - name: "encode values", - want: []byte{2, 0, 0, 0, 1, 0, 6, 0, 255, 255, 255, 255, 2, 0, 7, 0, 0, 0, 0, 0, 2, 0, 7, 0, 254, 255, 0, 0, 4, 0, 7, 0, 255, 255, 255, 255, 16, 0, 7, 0, 255, 255, 255, 255, 32, 0, 4, 0, 255, 255, 255, 255}, - args: []aclElement{ - { - aclSID: 8589934591, - Perm: 6, - }, - { - aclSID: 8589934592, - Perm: 7, - }, - { - aclSID: 8590000126, - Perm: 7, - }, - { - aclSID: 21474836479, - Perm: 7, - }, - { - aclSID: 73014444031, - Perm: 7, - }, - { - aclSID: 141733920767, - Perm: 4, - }, - }, - }, - { - name: "encode fail", - want: []byte{2, 0, 0, 0}, - args: []aclElement{}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - a := &acl{ - Version: 2, - List: tt.args, - } - if got := a.encode(); !reflect.DeepEqual(got, tt.want) { - t.Errorf("acl.encode() = %v, want %v", got, tt.want) - } - }) + {in: "\x02\x00", err: "wrong length"}, + {in: "", err: "wrong length"}, + } { + out, err := formatLinuxACL([]byte(c.in)) + if c.err == "" { + rtest.Equals(t, c.out, out) + } else { + rtest.Assert(t, err != nil, "wanted %q but got nil", c.err) + rtest.Equals(t, c.err, err.Error()) + } } } diff --git a/internal/dump/tar.go b/internal/dump/tar.go index df9ea429d..e8f34deb1 100644 --- a/internal/dump/tar.go +++ b/internal/dump/tar.go @@ -6,8 +6,8 @@ import ( "fmt" "os" "path/filepath" - "strings" + "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" ) @@ -104,21 +104,28 @@ func parseXattrs(xattrs []restic.ExtendedAttribute) map[string]string { tmpMap := make(map[string]string) for _, attr := range xattrs { - attrString := string(attr.Value) + // Check for Linux POSIX.1e ACLs. + // + // TODO support ACLs from other operating systems. + // FreeBSD ACLs have names "posix1e.acl_(access|default)", + // but their binary format may not match the Linux format. + aclKey := "" + switch attr.Name { + case "system.posix_acl_access": + aclKey = "SCHILY.acl.access" + case "system.posix_acl_default": + aclKey = "SCHILY.acl.default" + } - if strings.HasPrefix(attr.Name, "system.posix_acl_") { - na := acl{} - na.decode(attr.Value) - - if na.String() != "" { - if strings.Contains(attr.Name, "system.posix_acl_access") { - tmpMap["SCHILY.acl.access"] = na.String() - } else if strings.Contains(attr.Name, "system.posix_acl_default") { - tmpMap["SCHILY.acl.default"] = na.String() - } + if aclKey != "" { + text, err := formatLinuxACL(attr.Value) + if err != nil { + debug.Log("parsing Linux ACL: %v, skipping", err) + continue } + tmpMap[aclKey] = text } else { - tmpMap["SCHILY.xattr."+attr.Name] = attrString + tmpMap["SCHILY.xattr."+attr.Name] = string(attr.Value) } } diff --git a/internal/fs/fs_reader_command.go b/internal/fs/fs_reader_command.go new file mode 100644 index 000000000..3830e5811 --- /dev/null +++ b/internal/fs/fs_reader_command.go @@ -0,0 +1,97 @@ +package fs + +import ( + "bufio" + "context" + "fmt" + "io" + "os/exec" + + "github.com/restic/restic/internal/errors" +) + +// CommandReader wrap a command such that its standard output can be read using +// a io.ReadCloser. Close() waits for the command to terminate, reporting +// any error back to the caller. +type CommandReader struct { + cmd *exec.Cmd + stdout io.ReadCloser + + // cmd.Wait() must only be called once. Prevent duplicate executions in + // Read() and Close(). + waitHandled bool + + // alreadyClosedReadErr is the error that we should return if we try to + // read the pipe again after closing. This works around a Read() call that + // is issued after a previous Read() with `io.EOF` (but some bytes were + // read in the past). + alreadyClosedReadErr error +} + +func NewCommandReader(ctx context.Context, args []string, logOutput io.Writer) (*CommandReader, error) { + // Prepare command and stdout + command := exec.CommandContext(ctx, args[0], args[1:]...) + stdout, err := command.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("failed to setup stdout pipe: %w", err) + } + + // Use a Go routine to handle the stderr to avoid deadlocks + stderr, err := command.StderrPipe() + if err != nil { + return nil, fmt.Errorf("failed to setup stderr pipe: %w", err) + } + go func() { + sc := bufio.NewScanner(stderr) + for sc.Scan() { + _, _ = fmt.Fprintf(logOutput, "subprocess %v: %v\n", command.Args[0], sc.Text()) + } + }() + + if err := command.Start(); err != nil { + return nil, fmt.Errorf("failed to start command: %w", err) + } + + return &CommandReader{ + cmd: command, + stdout: stdout, + }, nil +} + +// Read populate the array with data from the process stdout. +func (fp *CommandReader) Read(p []byte) (int, error) { + if fp.alreadyClosedReadErr != nil { + return 0, fp.alreadyClosedReadErr + } + b, err := fp.stdout.Read(p) + + // If the error is io.EOF, the program terminated. We need to check the + // exit code here because, if the program terminated with no output, the + // error in `Close()` is ignored. + if errors.Is(err, io.EOF) { + fp.waitHandled = true + // check if the command terminated successfully, If not return the error. + if errw := fp.wait(); errw != nil { + err = errw + } + } + fp.alreadyClosedReadErr = err + return b, err +} + +func (fp *CommandReader) wait() error { + err := fp.cmd.Wait() + if err != nil { + // Use a fatal error to abort the snapshot. + return errors.Fatal(fmt.Errorf("command failed: %w", err).Error()) + } + return nil +} + +func (fp *CommandReader) Close() error { + if fp.waitHandled { + return nil + } + + return fp.wait() +} diff --git a/internal/fs/fs_reader_command_test.go b/internal/fs/fs_reader_command_test.go new file mode 100644 index 000000000..a9028544c --- /dev/null +++ b/internal/fs/fs_reader_command_test.go @@ -0,0 +1,48 @@ +package fs_test + +import ( + "bytes" + "context" + "io" + "strings" + "testing" + + "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/test" +) + +func TestCommandReaderSuccess(t *testing.T) { + reader, err := fs.NewCommandReader(context.TODO(), []string{"true"}, io.Discard) + test.OK(t, err) + + _, err = io.Copy(io.Discard, reader) + test.OK(t, err) + + test.OK(t, reader.Close()) +} + +func TestCommandReaderFail(t *testing.T) { + reader, err := fs.NewCommandReader(context.TODO(), []string{"false"}, io.Discard) + test.OK(t, err) + + _, err = io.Copy(io.Discard, reader) + test.Assert(t, err != nil, "missing error") +} + +func TestCommandReaderInvalid(t *testing.T) { + _, err := fs.NewCommandReader(context.TODO(), []string{"w54fy098hj7fy5twijouytfrj098y645wr"}, io.Discard) + test.Assert(t, err != nil, "missing error") +} + +func TestCommandReaderOutput(t *testing.T) { + reader, err := fs.NewCommandReader(context.TODO(), []string{"echo", "hello world"}, io.Discard) + test.OK(t, err) + + var buf bytes.Buffer + + _, err = io.Copy(&buf, reader) + test.OK(t, err) + test.OK(t, reader.Close()) + + test.Equals(t, "hello world", strings.TrimSpace(buf.String())) +} diff --git a/internal/fs/vss_windows.go b/internal/fs/vss_windows.go index bd82f4405..8c9b8942b 100644 --- a/internal/fs/vss_windows.go +++ b/internal/fs/vss_windows.go @@ -166,7 +166,7 @@ func (h HRESULT) Str() string { return "UNKNOWN" } -// VssError encapsulates errors retruned from calling VSS api. +// VssError encapsulates errors returned from calling VSS api. type vssError struct { text string hresult HRESULT @@ -190,7 +190,7 @@ func (e *vssError) Error() string { return fmt.Sprintf("VSS error: %s: %s (%#x)", e.text, e.hresult.Str(), e.hresult) } -// VssError encapsulates errors retruned from calling VSS api. +// VssError encapsulates errors returned from calling VSS api. type vssTextError struct { text string } @@ -615,7 +615,7 @@ func (vssAsync *IVSSAsync) QueryStatus() (HRESULT, uint32) { return HRESULT(result), state } -// WaitUntilAsyncFinished waits until either the async call is finshed or +// WaitUntilAsyncFinished waits until either the async call is finished or // the given timeout is reached. func (vssAsync *IVSSAsync) WaitUntilAsyncFinished(millis uint32) error { hresult := vssAsync.Wait(millis) @@ -858,7 +858,7 @@ func NewVssSnapshot( if err != nil { // After calling PrepareForBackup one needs to call AbortBackup() before releasing the VSS // instance for proper cleanup. - // It is not neccessary to call BackupComplete before releasing the VSS instance afterwards. + // It is not necessary to call BackupComplete before releasing the VSS instance afterwards. iVssBackupComponents.AbortBackup() iVssBackupComponents.Release() return VssSnapshot{}, err diff --git a/internal/fuse/dir.go b/internal/fuse/dir.go index 242b4b03e..c5aaf6f52 100644 --- a/internal/fuse/dir.go +++ b/internal/fuse/dir.go @@ -46,7 +46,7 @@ func newDir(root *Root, inode, parentInode uint64, node *restic.Node) (*dir, err }, nil } -// returing a wrapped context.Canceled error will instead result in returing +// returning a wrapped context.Canceled error will instead result in returning // an input / output error to the user. Thus unwrap the error to match the // expectations of bazil/fuse func unwrapCtxCanceled(err error) error { diff --git a/internal/fuse/file.go b/internal/fuse/file.go index aec39273a..6152c9122 100644 --- a/internal/fuse/file.go +++ b/internal/fuse/file.go @@ -142,8 +142,8 @@ func (f *openFile) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.R // Multiple goroutines may call service methods simultaneously; // the methods being called are responsible for appropriate synchronization. // - // However, no lock needed here as getBlobAt can be called conurrently - // (blobCache has it's own locking) + // However, no lock needed here as getBlobAt can be called concurrently + // (blobCache has its own locking) for i := startContent; remainingBytes > 0 && i < len(f.cumsize)-1; i++ { blob, err := f.getBlobAt(ctx, i) if err != nil { diff --git a/internal/fuse/fuse_test.go b/internal/fuse/fuse_test.go index ccdd2f774..0a121b986 100644 --- a/internal/fuse/fuse_test.go +++ b/internal/fuse/fuse_test.go @@ -73,7 +73,7 @@ func TestFuseFile(t *testing.T) { timestamp, err := time.Parse(time.RFC3339, "2017-01-24T10:42:56+01:00") rtest.OK(t, err) - restic.TestCreateSnapshot(t, repo, timestamp, 2, 0.1) + restic.TestCreateSnapshot(t, repo, timestamp, 2) sn := loadFirstSnapshot(t, repo) tree := loadTree(t, repo, *sn.Tree) @@ -180,7 +180,7 @@ func TestFuseDir(t *testing.T) { // Test top-level directories for their UID and GID. func TestTopUIDGID(t *testing.T) { repo := repository.TestRepository(t) - restic.TestCreateSnapshot(t, repo, time.Unix(1460289341, 207401672), 0, 0) + restic.TestCreateSnapshot(t, repo, time.Unix(1460289341, 207401672), 0) testTopUIDGID(t, Config{}, repo, uint32(os.Getuid()), uint32(os.Getgid())) testTopUIDGID(t, Config{OwnerIsRoot: true}, repo, 0, 0) diff --git a/internal/fuse/snapshots_dir.go b/internal/fuse/snapshots_dir.go index 61df3ad08..7369ea17a 100644 --- a/internal/fuse/snapshots_dir.go +++ b/internal/fuse/snapshots_dir.go @@ -110,9 +110,8 @@ func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error) return newSnapshotLink(d.root, inode, entry.linkTarget, entry.snapshot) } else if entry.snapshot != nil { return newDirFromSnapshot(d.root, inode, entry.snapshot) - } else { - return NewSnapshotsDir(d.root, inode, d.inode, d.dirStruct, d.prefix+"/"+name), nil } + return NewSnapshotsDir(d.root, inode, d.inode, d.dirStruct, d.prefix+"/"+name), nil } return nil, syscall.ENOENT diff --git a/internal/fuse/snapshots_dirstruct.go b/internal/fuse/snapshots_dirstruct.go index 3080d4de8..d40ae6298 100644 --- a/internal/fuse/snapshots_dirstruct.go +++ b/internal/fuse/snapshots_dirstruct.go @@ -295,7 +295,7 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error { } var snapshots restic.Snapshots - err := d.root.cfg.Filter.FindAll(ctx, d.root.repo.Backend(), d.root.repo, nil, func(id string, sn *restic.Snapshot, err error) error { + err := d.root.cfg.Filter.FindAll(ctx, d.root.repo, d.root.repo, nil, func(id string, sn *restic.Snapshot, err error) error { if sn != nil { snapshots = append(snapshots, sn) } @@ -328,7 +328,7 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error { return nil } - err = d.root.repo.LoadIndex(ctx) + err = d.root.repo.LoadIndex(ctx, nil) if err != nil { return err } diff --git a/internal/index/index_parallel.go b/internal/index/index_parallel.go index e7e46e88a..d505d756e 100644 --- a/internal/index/index_parallel.go +++ b/internal/index/index_parallel.go @@ -11,7 +11,7 @@ import ( // ForAllIndexes loads all index files in parallel and calls the given callback. // It is guaranteed that the function is not run concurrently. If the callback // returns an error, this function is cancelled and also returns that error. -func ForAllIndexes(ctx context.Context, repo restic.Repository, +func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.Repository, fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error { // decoding an index can take quite some time such that this can be both CPU- or IO-bound @@ -19,7 +19,7 @@ func ForAllIndexes(ctx context.Context, repo restic.Repository, workerCount := repo.Connections() + uint(runtime.GOMAXPROCS(0)) var m sync.Mutex - return restic.ParallelList(ctx, repo.Backend(), restic.IndexFile, workerCount, func(ctx context.Context, id restic.ID, size int64) error { + return restic.ParallelList(ctx, lister, restic.IndexFile, workerCount, func(ctx context.Context, id restic.ID, size int64) error { var err error var idx *Index oldFormat := false diff --git a/internal/index/index_parallel_test.go b/internal/index/index_parallel_test.go index 760374510..db4853e19 100644 --- a/internal/index/index_parallel_test.go +++ b/internal/index/index_parallel_test.go @@ -29,7 +29,7 @@ func TestRepositoryForAllIndexes(t *testing.T) { // check that all expected indexes are loaded without errors indexIDs := restic.NewIDSet() var indexErr error - rtest.OK(t, index.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { + rtest.OK(t, index.ForAllIndexes(context.TODO(), repo, repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { if err != nil { indexErr = err } @@ -42,7 +42,7 @@ func TestRepositoryForAllIndexes(t *testing.T) { // must failed with the returned error iterErr := errors.New("error to pass upwards") - err := index.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { + err := index.ForAllIndexes(context.TODO(), repo, repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { return iterErr }) diff --git a/internal/index/master_index.go b/internal/index/master_index.go index ca7c16135..073c9ace4 100644 --- a/internal/index/master_index.go +++ b/internal/index/master_index.go @@ -25,7 +25,7 @@ type MasterIndex struct { func NewMasterIndex() *MasterIndex { // Always add an empty final index, such that MergeFinalIndexes can merge into this. // Note that removing this index could lead to a race condition in the rare - // sitation that only two indexes exist which are saved and merged concurrently. + // situation that only two indexes exist which are saved and merged concurrently. idx := []*Index{NewIndex()} idx[0].Finalize() return &MasterIndex{idx: idx, pendingBlobs: restic.NewBlobSet()} diff --git a/internal/index/master_index_test.go b/internal/index/master_index_test.go index 5d12956bd..f76feb5fa 100644 --- a/internal/index/master_index_test.go +++ b/internal/index/master_index_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/index" @@ -43,9 +44,10 @@ func TestMasterIndex(t *testing.T) { blob12a := restic.PackedBlob{ PackID: restic.NewRandomID(), Blob: restic.Blob{ - BlobHandle: bhInIdx12, - Length: uint(crypto.CiphertextLength(123)), - Offset: 110, + BlobHandle: bhInIdx12, + Length: uint(crypto.CiphertextLength(123)), + Offset: 110, + UncompressedLength: 80, }, } @@ -116,7 +118,7 @@ func TestMasterIndex(t *testing.T) { size, found = mIdx.LookupSize(bhInIdx12) rtest.Equals(t, true, found) - rtest.Equals(t, uint(123), size) + rtest.Equals(t, uint(80), size) // test not in index found = mIdx.Has(restic.BlobHandle{ID: restic.NewRandomID(), Type: restic.TreeBlob}) @@ -340,11 +342,11 @@ var ( depth = 3 ) -func createFilledRepo(t testing.TB, snapshots int, dup float32, version uint) restic.Repository { +func createFilledRepo(t testing.TB, snapshots int, version uint) restic.Repository { repo := repository.TestRepositoryWithVersion(t, version) for i := 0; i < snapshots; i++ { - restic.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(i)*time.Second), depth, dup) + restic.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(i)*time.Second), depth) } return repo } @@ -354,9 +356,9 @@ func TestIndexSave(t *testing.T) { } func testIndexSave(t *testing.T, version uint) { - repo := createFilledRepo(t, 3, 0, version) + repo := createFilledRepo(t, 3, version) - err := repo.LoadIndex(context.TODO()) + err := repo.LoadIndex(context.TODO(), nil) if err != nil { t.Fatal(err) } @@ -368,7 +370,7 @@ func testIndexSave(t *testing.T, version uint) { for id := range obsoletes { t.Logf("remove index %v", id.Str()) - h := restic.Handle{Type: restic.IndexFile, Name: id.String()} + h := backend.Handle{Type: restic.IndexFile, Name: id.String()} err = repo.Backend().Remove(context.TODO(), h) if err != nil { t.Errorf("error removing index %v: %v", id, err) @@ -381,7 +383,7 @@ func testIndexSave(t *testing.T, version uint) { t.Error(err) } - hints, errs := checker.LoadIndex(context.TODO()) + hints, errs := checker.LoadIndex(context.TODO(), nil) for _, h := range hints { t.Logf("hint: %v\n", h) } diff --git a/internal/migrations/s3_layout.go b/internal/migrations/s3_layout.go index 78d2492d8..6b40013ee 100644 --- a/internal/migrations/s3_layout.go +++ b/internal/migrations/s3_layout.go @@ -6,6 +6,7 @@ import ( "os" "path" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/debug" @@ -21,26 +22,9 @@ func init() { // "default" layout. type S3Layout struct{} -func toS3Backend(b restic.Backend) *s3.Backend { - for b != nil { - if be, ok := b.(*s3.Backend); ok { - return be - } - - if be, ok := b.(restic.BackendUnwrapper); ok { - b = be.Unwrap() - } else { - // not the backend we're looking for - break - } - } - debug.Log("backend is not s3") - return nil -} - // Check tests whether the migration can be applied. func (m *S3Layout) Check(_ context.Context, repo restic.Repository) (bool, string, error) { - be := toS3Backend(repo.Backend()) + be := backend.AsBackend[*s3.Backend](repo.Backend()) if be == nil { debug.Log("backend is not s3") return false, "backend is not s3", nil @@ -80,8 +64,8 @@ func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l layout.Layou fmt.Fprintf(os.Stderr, "renaming file returned error: %v\n", err) } - return be.List(ctx, t, func(fi restic.FileInfo) error { - h := restic.Handle{Type: t, Name: fi.Name} + return be.List(ctx, t, func(fi backend.FileInfo) error { + h := backend.Handle{Type: t, Name: fi.Name} debug.Log("move %v", h) return retry(maxErrors, printErr, func() error { @@ -92,7 +76,7 @@ func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l layout.Layou // Apply runs the migration. func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error { - be := toS3Backend(repo.Backend()) + be := backend.AsBackend[*s3.Backend](repo.Backend()) if be == nil { debug.Log("backend is not s3") return errors.New("backend is not s3") diff --git a/internal/migrations/s3_layout_test.go b/internal/migrations/s3_layout_test.go deleted file mode 100644 index ad0eedea6..000000000 --- a/internal/migrations/s3_layout_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package migrations - -import ( - "testing" - - "github.com/restic/restic/internal/backend/mock" - "github.com/restic/restic/internal/backend/s3" - "github.com/restic/restic/internal/cache" - "github.com/restic/restic/internal/test" -) - -func TestS3UnwrapBackend(t *testing.T) { - // toS3Backend(b restic.Backend) *s3.Backend - - m := mock.NewBackend() - test.Assert(t, toS3Backend(m) == nil, "mock backend is not an s3 backend") - - // uninitialized fake backend for testing - s3 := &s3.Backend{} - test.Assert(t, toS3Backend(s3) == s3, "s3 was not returned") - - c := &cache.Backend{Backend: s3} - test.Assert(t, toS3Backend(c) == s3, "failed to unwrap s3 backend") - - c.Backend = m - test.Assert(t, toS3Backend(c) == nil, "a wrapped mock backend is not an s3 backend") -} diff --git a/internal/migrations/upgrade_repo_v2.go b/internal/migrations/upgrade_repo_v2.go index a81abc0e3..585d9e8c7 100644 --- a/internal/migrations/upgrade_repo_v2.go +++ b/internal/migrations/upgrade_repo_v2.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/restic" ) @@ -57,7 +58,7 @@ func (*UpgradeRepoV2) RepoCheck() bool { return true } func (*UpgradeRepoV2) upgrade(ctx context.Context, repo restic.Repository) error { - h := restic.Handle{Type: restic.ConfigFile} + h := backend.Handle{Type: backend.ConfigFile} if !repo.Backend().HasAtomicReplace() { // remove the original file for backends which do not support atomic overwriting @@ -85,7 +86,7 @@ func (m *UpgradeRepoV2) Apply(ctx context.Context, repo restic.Repository) error return fmt.Errorf("create temp dir failed: %w", err) } - h := restic.Handle{Type: restic.ConfigFile} + h := backend.Handle{Type: restic.ConfigFile} // read raw config file and save it to a temp dir, just in case var rawConfigFile []byte @@ -115,7 +116,7 @@ func (m *UpgradeRepoV2) Apply(ctx context.Context, repo restic.Repository) error // try contingency methods, reupload the original file _ = repo.Backend().Remove(ctx, h) - err = repo.Backend().Save(ctx, h, restic.NewByteReader(rawConfigFile, nil)) + err = repo.Backend().Save(ctx, h, backend.NewByteReader(rawConfigFile, nil)) if err != nil { repoError.ReuploadOldConfigError = err } diff --git a/internal/migrations/upgrade_repo_v2_test.go b/internal/migrations/upgrade_repo_v2_test.go index 96fc7788e..40153d3ca 100644 --- a/internal/migrations/upgrade_repo_v2_test.go +++ b/internal/migrations/upgrade_repo_v2_test.go @@ -7,9 +7,9 @@ import ( "sync" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) @@ -37,14 +37,14 @@ func TestUpgradeRepoV2(t *testing.T) { } type failBackend struct { - restic.Backend + backend.Backend mu sync.Mutex ConfigFileSavesUntilError uint } -func (be *failBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { - if h.Type != restic.ConfigFile { +func (be *failBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { + if h.Type != backend.ConfigFile { return be.Backend.Save(ctx, h, rd) } diff --git a/internal/pack/pack.go b/internal/pack/pack.go index 34ad9d071..211af7bfb 100644 --- a/internal/pack/pack.go +++ b/internal/pack/pack.go @@ -189,7 +189,7 @@ const ( // MaxHeaderSize is the max size of header including header-length field MaxHeaderSize = 16*1024*1024 + headerLengthSize - // number of header enries to download as part of header-length request + // number of header entries to download as part of header-length request eagerEntries = 15 ) diff --git a/internal/pack/pack_test.go b/internal/pack/pack_test.go index 3f7077390..76ff5c127 100644 --- a/internal/pack/pack_test.go +++ b/internal/pack/pack_test.go @@ -127,8 +127,8 @@ func TestUnpackReadSeeker(t *testing.T) { b := mem.New() id := restic.Hash(packData) - handle := restic.Handle{Type: restic.PackFile, Name: id.String()} - rtest.OK(t, b.Save(context.TODO(), handle, restic.NewByteReader(packData, b.Hasher()))) + handle := backend.Handle{Type: backend.PackFile, Name: id.String()} + rtest.OK(t, b.Save(context.TODO(), handle, backend.NewByteReader(packData, b.Hasher()))) verifyBlobs(t, bufs, k, backend.ReaderAt(context.TODO(), b, handle), packSize) } @@ -140,7 +140,7 @@ func TestShortPack(t *testing.T) { b := mem.New() id := restic.Hash(packData) - handle := restic.Handle{Type: restic.PackFile, Name: id.String()} - rtest.OK(t, b.Save(context.TODO(), handle, restic.NewByteReader(packData, b.Hasher()))) + handle := backend.Handle{Type: backend.PackFile, Name: id.String()} + rtest.OK(t, b.Save(context.TODO(), handle, backend.NewByteReader(packData, b.Hasher()))) verifyBlobs(t, bufs, k, backend.ReaderAt(context.TODO(), b, handle), packSize) } diff --git a/internal/repository/key.go b/internal/repository/key.go index fd20b8e5f..638d15d91 100644 --- a/internal/repository/key.go +++ b/internal/repository/key.go @@ -116,7 +116,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int, checked := 0 if len(keyHint) > 0 { - id, err := restic.Find(ctx, s.Backend(), restic.KeyFile, keyHint) + id, err := restic.Find(ctx, s, restic.KeyFile, keyHint) if err == nil { key, err := OpenKey(ctx, s, id, password) @@ -178,7 +178,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int, // LoadKey loads a key from the backend. func LoadKey(ctx context.Context, s *Repository, id restic.ID) (k *Key, err error) { - h := restic.Handle{Type: restic.KeyFile, Name: id.String()} + h := backend.Handle{Type: restic.KeyFile, Name: id.String()} data, err := backend.LoadAll(ctx, nil, s.be, h) if err != nil { return nil, err @@ -270,12 +270,12 @@ func AddKey(ctx context.Context, s *Repository, password, username, hostname str id := restic.Hash(buf) // store in repository and return - h := restic.Handle{ + h := backend.Handle{ Type: restic.KeyFile, Name: id.String(), } - err = s.be.Save(ctx, h, restic.NewByteReader(buf, s.be.Hasher())) + err = s.be.Save(ctx, h, backend.NewByteReader(buf, s.be.Hasher())) if err != nil { return nil, err } diff --git a/internal/repository/packer_manager.go b/internal/repository/packer_manager.go index 4422e3418..22eca0c2e 100644 --- a/internal/repository/packer_manager.go +++ b/internal/repository/packer_manager.go @@ -8,6 +8,7 @@ import ( "runtime" "sync" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/hashing" "github.com/restic/restic/internal/restic" @@ -38,7 +39,7 @@ type packerManager struct { packSize uint } -// newPackerManager returns an new packer manager which writes temporary files +// newPackerManager returns a new packer manager which writes temporary files // to a temporary directory func newPackerManager(key *crypto.Key, tpe restic.BlobType, packSize uint, queueFn func(ctx context.Context, t restic.BlobType, p *Packer) error) *packerManager { return &packerManager{ @@ -70,14 +71,19 @@ func (r *packerManager) SaveBlob(ctx context.Context, t restic.BlobType, id rest var err error packer := r.packer - if r.packer == nil { + // use separate packer if compressed length is larger than the packsize + // this speeds up the garbage collection of oversized blobs and reduces the cache size + // as the oversize blobs are only downloaded if necessary + if len(ciphertext) >= int(r.packSize) || r.packer == nil { packer, err = r.newPacker() if err != nil { return 0, err } + // don't store packer for oversized blob + if r.packer == nil { + r.packer = packer + } } - // remember packer - r.packer = packer // save ciphertext // Add only appends bytes in memory to avoid being a scaling bottleneck @@ -91,8 +97,10 @@ func (r *packerManager) SaveBlob(ctx context.Context, t restic.BlobType, id rest debug.Log("pack is not full enough (%d bytes)", packer.Size()) return size, nil } - // forget full packer - r.packer = nil + if packer == r.packer { + // forget full packer + r.packer = nil + } // call while holding lock to prevent findPacker from creating new packers if the uploaders are busy // else write the pack to the backend @@ -138,7 +146,7 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe // calculate sha256 hash in a second pass var rd io.Reader - rd, err = restic.NewFileReader(p.tmpfile, nil) + rd, err = backend.NewFileReader(p.tmpfile, nil) if err != nil { return err } @@ -156,12 +164,12 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe } id := restic.IDFromHash(hr.Sum(nil)) - h := restic.Handle{Type: restic.PackFile, Name: id.String(), ContainedBlobType: t} + h := backend.Handle{Type: backend.PackFile, Name: id.String(), IsMetadata: t.IsMetadata()} var beHash []byte if beHr != nil { beHash = beHr.Sum(nil) } - rrd, err := restic.NewFileReader(p.tmpfile, beHash) + rrd, err := backend.NewFileReader(p.tmpfile, beHash) if err != nil { return err } diff --git a/internal/repository/packer_manager_test.go b/internal/repository/packer_manager_test.go index 90f716e0d..8984073da 100644 --- a/internal/repository/packer_manager_test.go +++ b/internal/repository/packer_manager_test.go @@ -89,6 +89,24 @@ func testPackerManager(t testing.TB) int64 { return int64(bytes) } +func TestPackerManagerWithOversizeBlob(t *testing.T) { + packFiles := int(0) + sizeLimit := uint(512 * 1024) + pm := newPackerManager(crypto.NewRandomKey(), restic.DataBlob, sizeLimit, func(ctx context.Context, tp restic.BlobType, p *Packer) error { + packFiles++ + return nil + }) + + for _, i := range []uint{sizeLimit / 2, sizeLimit, sizeLimit / 3} { + _, err := pm.SaveBlob(context.TODO(), restic.DataBlob, restic.ID{}, make([]byte, i), 0) + test.OK(t, err) + } + test.OK(t, pm.Flush(context.TODO())) + + // oversized blob must be stored in a separate packfile + test.Equals(t, packFiles, 2) +} + func BenchmarkPackerManager(t *testing.B) { // Run testPackerManager if it hasn't run already, to set totalSize. once.Do(func() { diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index bb31eba77..20f0f2685 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -82,7 +83,7 @@ func createRandomWrongBlob(t testing.TB, repo restic.Repository) { } // selectBlobs splits the list of all blobs randomly into two lists. A blob -// will be contained in the firstone ith probability p. +// will be contained in the firstone with probability p. func selectBlobs(t *testing.T, repo restic.Repository, p float32) (list1, list2 restic.BlobSet) { list1 = restic.NewBlobSet() list2 = restic.NewBlobSet() @@ -157,7 +158,7 @@ func repack(t *testing.T, repo restic.Repository, packs restic.IDSet, blobs rest } for id := range repackedBlobs { - err = repo.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}) + err = repo.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()}) if err != nil { t.Fatal(err) } @@ -191,7 +192,7 @@ func rebuildIndex(t *testing.T, repo restic.Repository) { } err = repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error { - h := restic.Handle{ + h := backend.Handle{ Type: restic.IndexFile, Name: id.String(), } @@ -213,7 +214,7 @@ func reloadIndex(t *testing.T, repo restic.Repository) { t.Fatal(err) } - if err := repo.LoadIndex(context.TODO()); err != nil { + if err := repo.LoadIndex(context.TODO(), nil); err != nil { t.Fatalf("error loading new index: %v", err) } } diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 653c1f774..f78c55e1d 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -36,7 +36,7 @@ const MaxPackSize = 128 * 1024 * 1024 // Repository is used to access a repository in a backend. type Repository struct { - be restic.Backend + be backend.Backend cfg restic.Config key *crypto.Key keyID restic.ID @@ -109,7 +109,7 @@ func (c *CompressionMode) Type() string { } // New returns a new repository with backend be. -func New(be restic.Backend, opts Options) (*Repository, error) { +func New(be backend.Backend, opts Options) (*Repository, error) { if opts.Compression == CompressionInvalid { return nil, errors.New("invalid compression mode") } @@ -181,7 +181,7 @@ func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id res ctx, cancel := context.WithCancel(ctx) - h := restic.Handle{Type: t, Name: id.String()} + h := backend.Handle{Type: t, Name: id.String()} retriedInvalidData := false var dataErr error wr := new(bytes.Buffer) @@ -232,7 +232,7 @@ func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id res } type haver interface { - Has(restic.Handle) bool + Has(backend.Handle) bool } // sortCachedPacksFirst moves all cached pack files to the front of blobs. @@ -250,7 +250,7 @@ func sortCachedPacksFirst(cache haver, blobs []restic.PackedBlob) { noncached := make([]restic.PackedBlob, 0, len(blobs)/2) for _, blob := range blobs { - if cache.Has(restic.Handle{Type: restic.PackFile, Name: blob.PackID.String()}) { + if cache.Has(backend.Handle{Type: restic.PackFile, Name: blob.PackID.String()}) { cached = append(cached, blob) continue } @@ -284,7 +284,7 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic. } // load blob from pack - h := restic.Handle{Type: restic.PackFile, Name: blob.PackID.String(), ContainedBlobType: t} + h := backend.Handle{Type: restic.PackFile, Name: blob.PackID.String(), IsMetadata: t.IsMetadata()} switch { case cap(buf) < int(blob.Length): @@ -494,9 +494,9 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by } else { id = restic.Hash(ciphertext) } - h := restic.Handle{Type: t, Name: id.String()} + h := backend.Handle{Type: t, Name: id.String()} - err = r.be.Save(ctx, h, restic.NewByteReader(ciphertext, r.be.Hasher())) + err = r.be.Save(ctx, h, backend.NewByteReader(ciphertext, r.be.Hasher())) if err != nil { debug.Log("error saving blob %v: %v", h, err) return restic.ID{}, err @@ -561,7 +561,7 @@ func (r *Repository) flushPacks(ctx context.Context) error { } // Backend returns the backend for the repository. -func (r *Repository) Backend() restic.Backend { +func (r *Repository) Backend() backend.Backend { return r.be } @@ -581,15 +581,35 @@ func (r *Repository) SetIndex(i restic.MasterIndex) error { } // LoadIndex loads all index files from the backend in parallel and stores them -// in the master index. The first error that occurred is returned. -func (r *Repository) LoadIndex(ctx context.Context) error { +func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error { debug.Log("Loading index") - err := index.ForAllIndexes(ctx, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { + indexList, err := restic.MemorizeList(ctx, r, restic.IndexFile) + if err != nil { + return err + } + + if p != nil { + var numIndexFiles uint64 + err := indexList.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error { + numIndexFiles++ + return nil + }) + if err != nil { + return err + } + p.SetMax(numIndexFiles) + defer p.Done() + } + + err = index.ForAllIndexes(ctx, indexList, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { if err != nil { return err } r.idx.Insert(idx) + if p != nil { + p.Add(1) + } return nil }) @@ -747,7 +767,7 @@ func (r *Repository) Init(ctx context.Context, version uint, password string, ch return fmt.Errorf("repository version %v too low", version) } - _, err := r.be.Stat(ctx, restic.Handle{Type: restic.ConfigFile}) + _, err := r.be.Stat(ctx, backend.Handle{Type: restic.ConfigFile}) if err != nil && !r.be.IsNotExist(err) { return err } @@ -792,7 +812,7 @@ func (r *Repository) KeyID() restic.ID { // List runs fn for all files of type t in the repo. func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error { - return r.be.List(ctx, t, func(fi restic.FileInfo) error { + return r.be.List(ctx, t, func(fi backend.FileInfo) error { id, err := restic.ParseID(fi.Name) if err != nil { debug.Log("unable to parse %v as an ID", fi.Name) @@ -803,9 +823,9 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic } // ListPack returns the list of blobs saved in the pack id and the length of -// the the pack header. +// the pack header. func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) ([]restic.Blob, uint32, error) { - h := restic.Handle{Type: restic.PackFile, Name: id.String()} + h := backend.Handle{Type: restic.PackFile, Name: id.String()} return pack.List(r.Key(), backend.ReaderAt(ctx, r.Backend(), h), size) } @@ -855,7 +875,7 @@ func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte return newID, known, size, err } -type BackendLoadFn func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error +type BackendLoadFn func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error // Skip sections with more than 4MB unused blobs const maxUnusedRange = 4 * 1024 * 1024 @@ -896,7 +916,7 @@ func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, pack } func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { - h := restic.Handle{Type: restic.PackFile, Name: packID.String(), ContainedBlobType: restic.DataBlob} + h := backend.Handle{Type: restic.PackFile, Name: packID.String(), IsMetadata: false} dataStart := blobs[0].Offset dataEnd := blobs[len(blobs)-1].Offset + blobs[len(blobs)-1].Length @@ -912,7 +932,7 @@ func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, ctx, cancel := context.WithCancel(ctx) // stream blobs in pack err = beLoad(ctx, h, int(dataEnd-dataStart), int64(dataStart), func(rd io.Reader) error { - // prevent callbacks after cancelation + // prevent callbacks after cancellation if ctx.Err() != nil { return ctx.Err() } diff --git a/internal/repository/repository_internal_test.go b/internal/repository/repository_internal_test.go index e5ab6e5b7..d8e35b993 100644 --- a/internal/repository/repository_internal_test.go +++ b/internal/repository/repository_internal_test.go @@ -5,13 +5,14 @@ import ( "sort" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) -type mapcache map[restic.Handle]bool +type mapcache map[backend.Handle]bool -func (c mapcache) Has(h restic.Handle) bool { return c[h] } +func (c mapcache) Has(h backend.Handle) bool { return c[h] } func TestSortCachedPacksFirst(t *testing.T) { var ( @@ -27,15 +28,15 @@ func TestSortCachedPacksFirst(t *testing.T) { blobs[i] = restic.PackedBlob{PackID: id} if i%3 == 0 { - h := restic.Handle{Name: id.String(), Type: restic.PackFile} + h := backend.Handle{Name: id.String(), Type: backend.PackFile} cache[h] = true } } copy(sorted[:], blobs[:]) sort.SliceStable(sorted[:], func(i, j int) bool { - hi := restic.Handle{Type: restic.PackFile, Name: sorted[i].PackID.String()} - hj := restic.Handle{Type: restic.PackFile, Name: sorted[j].PackID.String()} + hi := backend.Handle{Type: backend.PackFile, Name: sorted[i].PackID.String()} + hj := backend.Handle{Type: backend.PackFile, Name: sorted[j].PackID.String()} return cache.Has(hi) && !cache.Has(hj) }) @@ -58,7 +59,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) { blobs[i] = restic.PackedBlob{PackID: id} if i%3 == 0 { - h := restic.Handle{Name: id.String(), Type: restic.PackFile} + h := backend.Handle{Name: id.String(), Type: backend.PackFile} cache[h] = true } } diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index f26bf46f2..bb8395436 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -16,6 +16,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/klauspost/compress/zstd" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/local" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/index" @@ -255,7 +256,7 @@ func TestRepositoryLoadIndex(t *testing.T) { defer cleanup() repo := repository.TestOpenLocal(t, repodir) - rtest.OK(t, repo.LoadIndex(context.TODO())) + rtest.OK(t, repo.LoadIndex(context.TODO(), nil)) } // loadIndex loads the index id from backend and returns it. @@ -278,13 +279,13 @@ func TestRepositoryLoadUnpackedBroken(t *testing.T) { data := rtest.Random(23, 12345) id := restic.Hash(data) - h := restic.Handle{Type: restic.IndexFile, Name: id.String()} + h := backend.Handle{Type: restic.IndexFile, Name: id.String()} // damage buffer data[0] ^= 0xff repo := repository.TestOpenLocal(t, repodir) // store broken file - err := repo.Backend().Save(context.TODO(), h, restic.NewByteReader(data, nil)) + err := repo.Backend().Save(context.TODO(), h, backend.NewByteReader(data, nil)) rtest.OK(t, err) // without a retry backend this will just return an error that the file is broken @@ -296,10 +297,10 @@ func TestRepositoryLoadUnpackedBroken(t *testing.T) { } type damageOnceBackend struct { - restic.Backend + backend.Backend } -func (be *damageOnceBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *damageOnceBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { // don't break the config file as we can't retry it if h.Type == restic.ConfigFile { return be.Backend.Load(ctx, h, length, offset, fn) @@ -324,7 +325,7 @@ func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) { err = repo.SearchKey(context.TODO(), rtest.TestPassword, 10, "") rtest.OK(t, err) - rtest.OK(t, repo.LoadIndex(context.TODO())) + rtest.OK(t, repo.LoadIndex(context.TODO(), nil)) } func BenchmarkLoadIndex(b *testing.B) { @@ -352,7 +353,7 @@ func benchmarkLoadIndex(b *testing.B, version uint) { rtest.OK(b, err) b.Logf("index saved as %v", id.Str()) - fi, err := repo.Backend().Stat(context.TODO(), restic.Handle{Type: restic.IndexFile, Name: id.String()}) + fi, err := repo.Backend().Stat(context.TODO(), backend.Handle{Type: restic.IndexFile, Name: id.String()}) rtest.OK(b, err) b.Logf("filesize is %v", fi.Size) @@ -522,13 +523,13 @@ func testStreamPack(t *testing.T, version uint) { case 2: compress = true default: - t.Fatal("test does not suport repository version", version) + t.Fatal("test does not support repository version", version) } packfileBlobs, packfile := buildPackfileWithoutHeader(blobSizes, &key, compress) loadCalls := 0 - load := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + load := func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { data := packfile if offset > int64(len(data)) { diff --git a/internal/repository/testing.go b/internal/repository/testing.go index 4936cc368..d79137425 100644 --- a/internal/repository/testing.go +++ b/internal/repository/testing.go @@ -6,6 +6,7 @@ import ( "os" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/local" "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/backend/retry" @@ -34,7 +35,7 @@ func TestUseLowSecurityKDFParameters(t logger) { } // TestBackend returns a fully configured in-memory backend. -func TestBackend(_ testing.TB) restic.Backend { +func TestBackend(_ testing.TB) backend.Backend { return mem.New() } @@ -43,7 +44,7 @@ const TestChunkerPol = chunker.Pol(0x3DA3358B4DC173) // TestRepositoryWithBackend returns a repository initialized with a test // password. If be is nil, an in-memory backend is used. A constant polynomial // is used for the chunker and low-security test parameters. -func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint) restic.Repository { +func TestRepositoryWithBackend(t testing.TB, be backend.Backend, version uint) restic.Repository { t.Helper() TestUseLowSecurityKDFParameters(t) restic.TestDisableCheckPolynomial(t) @@ -98,7 +99,7 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository { // TestOpenLocal opens a local repository. func TestOpenLocal(t testing.TB, dir string) (r restic.Repository) { - var be restic.Backend + var be backend.Backend be, err := local.Open(context.TODO(), local.Config{Path: dir, Connections: 2}) if err != nil { t.Fatal(err) diff --git a/internal/restic/backend_find.go b/internal/restic/backend_find.go index 7c78b3355..a6eacabd0 100644 --- a/internal/restic/backend_find.go +++ b/internal/restic/backend_find.go @@ -3,8 +3,6 @@ package restic import ( "context" "fmt" - - "github.com/restic/restic/internal/debug" ) // A MultipleIDMatchesError is returned by Find() when multiple IDs with a @@ -32,15 +30,9 @@ func Find(ctx context.Context, be Lister, t FileType, prefix string) (ID, error) ctx, cancel := context.WithCancel(ctx) defer cancel() - err := be.List(ctx, t, func(fi FileInfo) error { - // ignore filename which are not an id - id, err := ParseID(fi.Name) - if err != nil { - debug.Log("unable to parse %v as an ID", fi.Name) - return nil - } - - if len(fi.Name) >= len(prefix) && prefix == fi.Name[:len(prefix)] { + err := be.List(ctx, t, func(id ID, size int64) error { + name := id.String() + if len(name) >= len(prefix) && prefix == name[:len(prefix)] { if match.IsNull() { match = id } else { diff --git a/internal/restic/backend_find_test.go b/internal/restic/backend_find_test.go index cbd5e7f48..5f020fda6 100644 --- a/internal/restic/backend_find_test.go +++ b/internal/restic/backend_find_test.go @@ -1,37 +1,31 @@ -package restic +package restic_test import ( "context" "strings" "testing" + + "github.com/restic/restic/internal/restic" ) -type mockBackend struct { - list func(context.Context, FileType, func(FileInfo) error) error -} - -func (m mockBackend) List(ctx context.Context, t FileType, fn func(FileInfo) error) error { - return m.list(ctx, t, fn) -} - -var samples = IDs{ - TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff"), - TestParseID("20bdc1402a6fc9b633ccd578c4a92d0f4ef1a457fa2e16c596bc73fb409d6cc0"), - TestParseID("20bdc1402a6fc9b633ffffffffffffffffffffffffffffffffffffffffffffff"), - TestParseID("20ff988befa5fc40350f00d531a767606efefe242c837aaccb80673f286be53d"), - TestParseID("326cb59dfe802304f96ee9b5b9af93bdee73a30f53981e5ec579aedb6f1d0f07"), - TestParseID("86b60b9594d1d429c4aa98fa9562082cabf53b98c7dc083abe5dae31074dd15a"), - TestParseID("96c8dbe225079e624b5ce509f5bd817d1453cd0a85d30d536d01b64a8669aeae"), - TestParseID("fa31d65b87affcd167b119e9d3d2a27b8236ca4836cb077ed3e96fcbe209b792"), +var samples = restic.IDs{ + restic.TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff"), + restic.TestParseID("20bdc1402a6fc9b633ccd578c4a92d0f4ef1a457fa2e16c596bc73fb409d6cc0"), + restic.TestParseID("20bdc1402a6fc9b633ffffffffffffffffffffffffffffffffffffffffffffff"), + restic.TestParseID("20ff988befa5fc40350f00d531a767606efefe242c837aaccb80673f286be53d"), + restic.TestParseID("326cb59dfe802304f96ee9b5b9af93bdee73a30f53981e5ec579aedb6f1d0f07"), + restic.TestParseID("86b60b9594d1d429c4aa98fa9562082cabf53b98c7dc083abe5dae31074dd15a"), + restic.TestParseID("96c8dbe225079e624b5ce509f5bd817d1453cd0a85d30d536d01b64a8669aeae"), + restic.TestParseID("fa31d65b87affcd167b119e9d3d2a27b8236ca4836cb077ed3e96fcbe209b792"), } func TestFind(t *testing.T) { list := samples - m := mockBackend{} - m.list = func(ctx context.Context, t FileType, fn func(FileInfo) error) error { + m := &ListHelper{} + m.ListFn = func(ctx context.Context, t restic.FileType, fn func(id restic.ID, size int64) error) error { for _, id := range list { - err := fn(FileInfo{Name: id.String()}) + err := fn(id, 0) if err != nil { return err } @@ -39,17 +33,17 @@ func TestFind(t *testing.T) { return nil } - f, err := Find(context.TODO(), m, SnapshotFile, "20bdc1402a6fc9b633aa") + f, err := restic.Find(context.TODO(), m, restic.SnapshotFile, "20bdc1402a6fc9b633aa") if err != nil { t.Error(err) } - expectedMatch := TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff") + expectedMatch := restic.TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff") if f != expectedMatch { t.Errorf("Wrong match returned want %s, got %s", expectedMatch, f) } - f, err = Find(context.TODO(), m, SnapshotFile, "NotAPrefix") - if _, ok := err.(*NoIDByPrefixError); !ok || !strings.Contains(err.Error(), "NotAPrefix") { + f, err = restic.Find(context.TODO(), m, restic.SnapshotFile, "NotAPrefix") + if _, ok := err.(*restic.NoIDByPrefixError); !ok || !strings.Contains(err.Error(), "NotAPrefix") { t.Error("Expected no snapshots to be found.") } if !f.IsNull() { @@ -58,8 +52,8 @@ func TestFind(t *testing.T) { // Try to match with a prefix longer than any ID. extraLengthID := samples[0].String() + "f" - f, err = Find(context.TODO(), m, SnapshotFile, extraLengthID) - if _, ok := err.(*NoIDByPrefixError); !ok || !strings.Contains(err.Error(), extraLengthID) { + f, err = restic.Find(context.TODO(), m, restic.SnapshotFile, extraLengthID) + if _, ok := err.(*restic.NoIDByPrefixError); !ok || !strings.Contains(err.Error(), extraLengthID) { t.Errorf("Wrong error %v for no snapshots matched", err) } if !f.IsNull() { @@ -67,8 +61,8 @@ func TestFind(t *testing.T) { } // Use a prefix that will match the prefix of multiple Ids in `samples`. - f, err = Find(context.TODO(), m, SnapshotFile, "20bdc140") - if _, ok := err.(*MultipleIDMatchesError); !ok { + f, err = restic.Find(context.TODO(), m, restic.SnapshotFile, "20bdc140") + if _, ok := err.(*restic.MultipleIDMatchesError); !ok { t.Errorf("Wrong error %v for multiple snapshots", err) } if !f.IsNull() { diff --git a/internal/restic/blob.go b/internal/restic/blob.go index 260f40fde..3a6872af3 100644 --- a/internal/restic/blob.go +++ b/internal/restic/blob.go @@ -75,6 +75,15 @@ func (t BlobType) String() string { return fmt.Sprintf("", t) } +func (t BlobType) IsMetadata() bool { + switch t { + case TreeBlob: + return true + default: + return false + } +} + // MarshalJSON encodes the BlobType into JSON. func (t BlobType) MarshalJSON() ([]byte, error) { switch t { diff --git a/internal/restic/blob_set.go b/internal/restic/blob_set.go index acacd57d4..928d0fb68 100644 --- a/internal/restic/blob_set.go +++ b/internal/restic/blob_set.go @@ -1,6 +1,10 @@ package restic -import "sort" +import ( + "fmt" + "sort" + "strings" +) // BlobSet is a set of blobs. type BlobSet map[BlobHandle]struct{} @@ -103,11 +107,27 @@ func (s BlobSet) List() BlobHandles { return list } +// String produces a human-readable representation of ids. +// It is meant for producing error messages, +// so it only returns a summary if ids is long. func (s BlobSet) String() string { - str := s.List().String() - if len(str) < 2 { - return "{}" - } + const maxelems = 10 - return "{" + str[1:len(str)-1] + "}" + sb := new(strings.Builder) + sb.WriteByte('{') + n := 0 + for k := range s { + if n != 0 { + sb.WriteByte(' ') + } + sb.WriteString(k.String()) + + if n++; n == maxelems { + fmt.Fprintf(sb, " (%d more)", len(s)-n-1) + break + } + } + sb.WriteByte('}') + + return sb.String() } diff --git a/internal/restic/blob_set_test.go b/internal/restic/blob_set_test.go new file mode 100644 index 000000000..e26b48fe9 --- /dev/null +++ b/internal/restic/blob_set_test.go @@ -0,0 +1,32 @@ +package restic + +import ( + "math/rand" + "regexp" + "testing" + + rtest "github.com/restic/restic/internal/test" +) + +func TestBlobSetString(t *testing.T) { + s := NewBlobSet() + + rtest.Equals(t, "{}", s.String()) + + id, _ := ParseID( + "1111111111111111111111111111111111111111111111111111111111111111") + s.Insert(BlobHandle{ID: id, Type: TreeBlob}) + rtest.Equals(t, "{}", s.String()) + + var h BlobHandle + for i := 0; i < 100; i++ { + h.Type = DataBlob + _, _ = rand.Read(h.ID[:]) + s.Insert(h) + } + + r := regexp.MustCompile( + `^{(?:<(?:data|tree)/[0-9a-f]{8}> ){10}\(90 more\)}$`) + str := s.String() + rtest.Assert(t, r.MatchString(str), "%q doesn't match pattern", str) +} diff --git a/internal/restic/counted_blob_set_test.go b/internal/restic/counted_blob_set_test.go index 681751e91..edd39e65b 100644 --- a/internal/restic/counted_blob_set_test.go +++ b/internal/restic/counted_blob_set_test.go @@ -13,7 +13,7 @@ func TestCountedBlobSet(t *testing.T) { test.Equals(t, bs.List(), restic.BlobHandles{}) bh := restic.NewRandomBlobHandle() - // check non existant + // check non existent test.Equals(t, bs.Has(bh), false) // test insert diff --git a/internal/restic/find_test.go b/internal/restic/find_test.go index 234561b6d..1ae30ded9 100644 --- a/internal/restic/find_test.go +++ b/internal/restic/find_test.go @@ -88,7 +88,7 @@ func TestFindUsedBlobs(t *testing.T) { var snapshots []*restic.Snapshot for i := 0; i < findTestSnapshots; i++ { - sn := restic.TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth, 0) + sn := restic.TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth) t.Logf("snapshot %v saved, tree %v", sn.ID().Str(), sn.Tree.Str()) snapshots = append(snapshots, sn) } @@ -131,7 +131,7 @@ func TestMultiFindUsedBlobs(t *testing.T) { var snapshotTrees restic.IDs for i := 0; i < findTestSnapshots; i++ { - sn := restic.TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth, 0) + sn := restic.TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth) t.Logf("snapshot %v saved, tree %v", sn.ID().Str(), sn.Tree.Str()) snapshotTrees = append(snapshotTrees, *sn.Tree) } @@ -177,7 +177,7 @@ func (r ForbiddenRepo) Connections() uint { func TestFindUsedBlobsSkipsSeenBlobs(t *testing.T) { repo := repository.TestRepository(t) - snapshot := restic.TestCreateSnapshot(t, repo, findTestTime, findTestDepth, 0) + snapshot := restic.TestCreateSnapshot(t, repo, findTestTime, findTestDepth) t.Logf("snapshot %v saved, tree %v", snapshot.ID().Str(), snapshot.Tree.Str()) usedBlobs := restic.NewBlobSet() @@ -195,7 +195,7 @@ func TestFindUsedBlobsSkipsSeenBlobs(t *testing.T) { func BenchmarkFindUsedBlobs(b *testing.B) { repo := repository.TestRepository(b) - sn := restic.TestCreateSnapshot(b, repo, findTestTime, findTestDepth, 0) + sn := restic.TestCreateSnapshot(b, repo, findTestTime, findTestDepth) b.ResetTimer() diff --git a/internal/restic/lister.go b/internal/restic/lister.go new file mode 100644 index 000000000..23da30b7d --- /dev/null +++ b/internal/restic/lister.go @@ -0,0 +1,52 @@ +package restic + +import ( + "context" + "fmt" +) + +type fileInfo struct { + id ID + size int64 +} + +type memorizedLister struct { + fileInfos []fileInfo + tpe FileType +} + +func (m *memorizedLister) List(ctx context.Context, t FileType, fn func(ID, int64) error) error { + if t != m.tpe { + return fmt.Errorf("filetype mismatch, expected %s got %s", m.tpe, t) + } + for _, fi := range m.fileInfos { + if ctx.Err() != nil { + break + } + err := fn(fi.id, fi.size) + if err != nil { + return err + } + } + return ctx.Err() +} + +func MemorizeList(ctx context.Context, be Lister, t FileType) (Lister, error) { + if _, ok := be.(*memorizedLister); ok { + return be, nil + } + + var fileInfos []fileInfo + err := be.List(ctx, t, func(id ID, size int64) error { + fileInfos = append(fileInfos, fileInfo{id, size}) + return nil + }) + if err != nil { + return nil, err + } + + return &memorizedLister{ + fileInfos: fileInfos, + tpe: t, + }, nil +} diff --git a/internal/restic/lister_test.go b/internal/restic/lister_test.go new file mode 100644 index 000000000..245a5d3da --- /dev/null +++ b/internal/restic/lister_test.go @@ -0,0 +1,68 @@ +package restic_test + +import ( + "context" + "fmt" + "testing" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" +) + +type ListHelper struct { + ListFn func(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error +} + +func (l *ListHelper) List(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error { + return l.ListFn(ctx, t, fn) +} + +func TestMemoizeList(t *testing.T) { + // setup backend to serve as data source for memoized list + be := &ListHelper{} + + type FileInfo struct { + ID restic.ID + Size int64 + } + files := []FileInfo{ + {ID: restic.NewRandomID(), Size: 42}, + {ID: restic.NewRandomID(), Size: 45}, + } + be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error { + for _, fi := range files { + if err := fn(fi.ID, fi.Size); err != nil { + return err + } + } + return nil + } + + mem, err := restic.MemorizeList(context.TODO(), be, backend.SnapshotFile) + rtest.OK(t, err) + + err = mem.List(context.TODO(), backend.IndexFile, func(id restic.ID, size int64) error { + t.Fatal("file type mismatch") + return nil // the memoized lister must return an error by itself + }) + rtest.Assert(t, err != nil, "missing error on file typ mismatch") + + var memFiles []FileInfo + err = mem.List(context.TODO(), backend.SnapshotFile, func(id restic.ID, size int64) error { + memFiles = append(memFiles, FileInfo{ID: id, Size: size}) + return nil + }) + rtest.OK(t, err) + rtest.Equals(t, files, memFiles) +} + +func TestMemoizeListError(t *testing.T) { + // setup backend to serve as data source for memoized list + be := &ListHelper{} + be.ListFn = func(ctx context.Context, t backend.FileType, fn func(restic.ID, int64) error) error { + return fmt.Errorf("list error") + } + _, err := restic.MemorizeList(context.TODO(), be, backend.SnapshotFile) + rtest.Assert(t, err != nil, "missing error on list error") +} diff --git a/internal/restic/lock.go b/internal/restic/lock.go index d500c019a..175cf6188 100644 --- a/internal/restic/lock.go +++ b/internal/restic/lock.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/debug" @@ -81,6 +82,8 @@ func IsInvalidLock(err error) bool { return errors.As(err, &e) } +var ErrRemovedLock = errors.New("lock file was removed in the meantime") + // NewLock returns a new, non-exclusive lock for the repository. If an // exclusive lock is already held by another process, it returns an error // that satisfies IsAlreadyLocked. @@ -211,7 +214,7 @@ func (l *Lock) Unlock() error { return nil } - return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: l.lockID.String()}) + return l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: l.lockID.String()}) } var StaleLockTimeout = 30 * time.Minute @@ -271,7 +274,69 @@ func (l *Lock) Refresh(ctx context.Context) error { oldLockID := l.lockID l.lockID = &id - return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: oldLockID.String()}) + return l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: oldLockID.String()}) +} + +// RefreshStaleLock is an extended variant of Refresh that can also refresh stale lock files. +func (l *Lock) RefreshStaleLock(ctx context.Context) error { + debug.Log("refreshing stale lock %v", l.lockID) + // refreshing a stale lock is possible if it still exists and continues to do + // so until after creating a new lock. The initial check avoids creating a new + // lock file if this lock was already removed in the meantime. + exists, err := l.checkExistence(ctx) + if err != nil { + return err + } else if !exists { + return ErrRemovedLock + } + + l.lock.Lock() + l.Time = time.Now() + l.lock.Unlock() + id, err := l.createLock(ctx) + if err != nil { + return err + } + + time.Sleep(waitBeforeLockCheck) + + exists, err = l.checkExistence(ctx) + if err != nil { + // cleanup replacement lock + _ = l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: id.String()}) + return err + } + + if !exists { + // cleanup replacement lock + _ = l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: id.String()}) + return ErrRemovedLock + } + + l.lock.Lock() + defer l.lock.Unlock() + + debug.Log("new lock ID %v", id) + oldLockID := l.lockID + l.lockID = &id + + return l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: oldLockID.String()}) +} + +func (l *Lock) checkExistence(ctx context.Context) (bool, error) { + l.lock.Lock() + defer l.lock.Unlock() + + exists := false + + err := l.repo.Backend().List(ctx, LockFile, func(fi backend.FileInfo) error { + if fi.Name == l.lockID.String() { + exists = true + } + return nil + }) + + return exists, err } func (l *Lock) String() string { @@ -323,7 +388,7 @@ func RemoveStaleLocks(ctx context.Context, repo Repository) (uint, error) { } if lock.Stale() { - err = repo.Backend().Remove(ctx, Handle{Type: LockFile, Name: id.String()}) + err = repo.Backend().Remove(ctx, backend.Handle{Type: LockFile, Name: id.String()}) if err == nil { processed++ } @@ -338,8 +403,8 @@ func RemoveStaleLocks(ctx context.Context, repo Repository) (uint, error) { // RemoveAllLocks removes all locks forcefully. func RemoveAllLocks(ctx context.Context, repo Repository) (uint, error) { var processed uint32 - err := ParallelList(ctx, repo.Backend(), LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error { - err := repo.Backend().Remove(ctx, Handle{Type: LockFile, Name: id.String()}) + err := ParallelList(ctx, repo, LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error { + err := repo.Backend().Remove(ctx, backend.Handle{Type: LockFile, Name: id.String()}) if err == nil { atomic.AddUint32(&processed, 1) } @@ -356,7 +421,7 @@ func ForAllLocks(ctx context.Context, repo Repository, excludeID *ID, fn func(ID var m sync.Mutex // For locks decoding is nearly for free, thus just assume were only limited by IO - return ParallelList(ctx, repo.Backend(), LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error { + return ParallelList(ctx, repo, LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error { if excludeID != nil && id.Equal(*excludeID) { return nil } diff --git a/internal/restic/lock_test.go b/internal/restic/lock_test.go index 2d14499bd..faf3f3593 100644 --- a/internal/restic/lock_test.go +++ b/internal/restic/lock_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -16,6 +17,7 @@ import ( func TestLock(t *testing.T) { repo := repository.TestRepository(t) + restic.TestSetLockTimeout(t, 5*time.Millisecond) lock, err := restic.NewLock(context.TODO(), repo) rtest.OK(t, err) @@ -25,6 +27,7 @@ func TestLock(t *testing.T) { func TestDoubleUnlock(t *testing.T) { repo := repository.TestRepository(t) + restic.TestSetLockTimeout(t, 5*time.Millisecond) lock, err := restic.NewLock(context.TODO(), repo) rtest.OK(t, err) @@ -38,6 +41,7 @@ func TestDoubleUnlock(t *testing.T) { func TestMultipleLock(t *testing.T) { repo := repository.TestRepository(t) + restic.TestSetLockTimeout(t, 5*time.Millisecond) lock1, err := restic.NewLock(context.TODO(), repo) rtest.OK(t, err) @@ -50,10 +54,10 @@ func TestMultipleLock(t *testing.T) { } type failLockLoadingBackend struct { - restic.Backend + backend.Backend } -func (be *failLockLoadingBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *failLockLoadingBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { if h.Type == restic.LockFile { return fmt.Errorf("error loading lock") } @@ -63,6 +67,7 @@ func (be *failLockLoadingBackend) Load(ctx context.Context, h restic.Handle, len func TestMultipleLockFailure(t *testing.T) { be := &failLockLoadingBackend{Backend: mem.New()} repo := repository.TestRepositoryWithBackend(t, be, 0) + restic.TestSetLockTimeout(t, 5*time.Millisecond) lock1, err := restic.NewLock(context.TODO(), repo) rtest.OK(t, err) @@ -83,6 +88,7 @@ func TestLockExclusive(t *testing.T) { func TestLockOnExclusiveLockedRepo(t *testing.T) { repo := repository.TestRepository(t) + restic.TestSetLockTimeout(t, 5*time.Millisecond) elock, err := restic.NewExclusiveLock(context.TODO(), repo) rtest.OK(t, err) @@ -99,6 +105,7 @@ func TestLockOnExclusiveLockedRepo(t *testing.T) { func TestExclusiveLockOnLockedRepo(t *testing.T) { repo := repository.TestRepository(t) + restic.TestSetLockTimeout(t, 5*time.Millisecond) elock, err := restic.NewLock(context.TODO(), repo) rtest.OK(t, err) @@ -124,7 +131,7 @@ func createFakeLock(repo restic.Repository, t time.Time, pid int) (restic.ID, er } func removeLock(repo restic.Repository, id restic.ID) error { - h := restic.Handle{Type: restic.LockFile, Name: id.String()} + h := backend.Handle{Type: restic.LockFile, Name: id.String()} return repo.Backend().Remove(context.TODO(), h) } @@ -185,7 +192,7 @@ func TestLockStale(t *testing.T) { } func lockExists(repo restic.Repository, t testing.TB, id restic.ID) bool { - h := restic.Handle{Type: restic.LockFile, Name: id.String()} + h := backend.Handle{Type: restic.LockFile, Name: id.String()} _, err := repo.Backend().Stat(context.TODO(), h) if err != nil && !repo.Backend().IsNotExist(err) { t.Fatal(err) @@ -247,15 +254,10 @@ func TestRemoveAllLocks(t *testing.T) { 3, processed) } -func TestLockRefresh(t *testing.T) { - repo := repository.TestRepository(t) - - lock, err := restic.NewLock(context.TODO(), repo) - rtest.OK(t, err) - time0 := lock.Time - +func checkSingleLock(t *testing.T, repo restic.Repository) restic.ID { + t.Helper() var lockID *restic.ID - err = repo.List(context.TODO(), restic.LockFile, func(id restic.ID, size int64) error { + err := repo.List(context.TODO(), restic.LockFile, func(id restic.ID, size int64) error { if lockID != nil { t.Error("more than one lock found") } @@ -265,27 +267,59 @@ func TestLockRefresh(t *testing.T) { if err != nil { t.Fatal(err) } + if lockID == nil { + t.Fatal("no lock found") + } + return *lockID +} + +func testLockRefresh(t *testing.T, refresh func(lock *restic.Lock) error) { + repo := repository.TestRepository(t) + restic.TestSetLockTimeout(t, 5*time.Millisecond) + + lock, err := restic.NewLock(context.TODO(), repo) + rtest.OK(t, err) + time0 := lock.Time + + lockID := checkSingleLock(t, repo) time.Sleep(time.Millisecond) - rtest.OK(t, lock.Refresh(context.TODO())) + rtest.OK(t, refresh(lock)) - var lockID2 *restic.ID - err = repo.List(context.TODO(), restic.LockFile, func(id restic.ID, size int64) error { - if lockID2 != nil { - t.Error("more than one lock found") - } - lockID2 = &id - return nil - }) - if err != nil { - t.Fatal(err) - } + lockID2 := checkSingleLock(t, repo) - rtest.Assert(t, !lockID.Equal(*lockID2), + rtest.Assert(t, !lockID.Equal(lockID2), "expected a new ID after lock refresh, got the same") - lock2, err := restic.LoadLock(context.TODO(), repo, *lockID2) + lock2, err := restic.LoadLock(context.TODO(), repo, lockID2) rtest.OK(t, err) rtest.Assert(t, lock2.Time.After(time0), "expected a later timestamp after lock refresh") rtest.OK(t, lock.Unlock()) } + +func TestLockRefresh(t *testing.T) { + testLockRefresh(t, func(lock *restic.Lock) error { + return lock.Refresh(context.TODO()) + }) +} + +func TestLockRefreshStale(t *testing.T) { + testLockRefresh(t, func(lock *restic.Lock) error { + return lock.RefreshStaleLock(context.TODO()) + }) +} + +func TestLockRefreshStaleMissing(t *testing.T) { + repo := repository.TestRepository(t) + restic.TestSetLockTimeout(t, 5*time.Millisecond) + + lock, err := restic.NewLock(context.TODO(), repo) + rtest.OK(t, err) + lockID := checkSingleLock(t, repo) + + // refresh must fail if lock was removed + rtest.OK(t, repo.Backend().Remove(context.TODO(), backend.Handle{Type: restic.LockFile, Name: lockID.String()})) + time.Sleep(time.Millisecond) + err = lock.RefreshStaleLock(context.TODO()) + rtest.Assert(t, err == restic.ErrRemovedLock, "unexpected error, expected %v, got %v", restic.ErrRemovedLock, err) +} diff --git a/internal/restic/node.go b/internal/restic/node.go index f2d9f2315..edb49bfca 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -10,6 +10,7 @@ import ( "sync" "syscall" "time" + "unicode/utf8" "github.com/restic/restic/internal/errors" @@ -27,21 +28,26 @@ type ExtendedAttribute struct { // Node is a file, directory or other item in a backup. type Node struct { - Name string `json:"name"` - Type string `json:"type"` - Mode os.FileMode `json:"mode,omitempty"` - ModTime time.Time `json:"mtime,omitempty"` - AccessTime time.Time `json:"atime,omitempty"` - ChangeTime time.Time `json:"ctime,omitempty"` - UID uint32 `json:"uid"` - GID uint32 `json:"gid"` - User string `json:"user,omitempty"` - Group string `json:"group,omitempty"` - Inode uint64 `json:"inode,omitempty"` - DeviceID uint64 `json:"device_id,omitempty"` // device id of the file, stat.st_dev - Size uint64 `json:"size,omitempty"` - Links uint64 `json:"links,omitempty"` - LinkTarget string `json:"linktarget,omitempty"` + Name string `json:"name"` + Type string `json:"type"` + Mode os.FileMode `json:"mode,omitempty"` + ModTime time.Time `json:"mtime,omitempty"` + AccessTime time.Time `json:"atime,omitempty"` + ChangeTime time.Time `json:"ctime,omitempty"` + UID uint32 `json:"uid"` + GID uint32 `json:"gid"` + User string `json:"user,omitempty"` + Group string `json:"group,omitempty"` + Inode uint64 `json:"inode,omitempty"` + DeviceID uint64 `json:"device_id,omitempty"` // device id of the file, stat.st_dev + Size uint64 `json:"size,omitempty"` + Links uint64 `json:"links,omitempty"` + LinkTarget string `json:"linktarget,omitempty"` + // implicitly base64-encoded field. Only used while encoding, `linktarget_raw` will overwrite LinkTarget if present. + // This allows storing arbitrary byte-sequences, which are possible as symlink targets on unix systems, + // as LinkTarget without breaking backwards-compatibility. + // Must only be set of the linktarget cannot be encoded as valid utf8. + LinkTargetRaw []byte `json:"linktarget_raw,omitempty"` ExtendedAttributes []ExtendedAttribute `json:"extended_attributes,omitempty"` Device uint64 `json:"device,omitempty"` // in case of Type == "dev", stat.st_rdev Content IDs `json:"content"` @@ -344,6 +350,13 @@ func (node Node) MarshalJSON() ([]byte, error) { nj := nodeJSON(node) name := strconv.Quote(node.Name) nj.Name = name[1 : len(name)-1] + if nj.LinkTargetRaw != nil { + panic("LinkTargetRaw must not be set manually") + } + if !utf8.ValidString(node.LinkTarget) { + // store raw bytes if invalid utf8 + nj.LinkTargetRaw = []byte(node.LinkTarget) + } return json.Marshal(nj) } @@ -358,7 +371,14 @@ func (node *Node) UnmarshalJSON(data []byte) error { } nj.Name, err = strconv.Unquote(`"` + nj.Name + `"`) - return errors.Wrap(err, "Unquote") + if err != nil { + return errors.Wrap(err, "Unquote") + } + if nj.LinkTargetRaw != nil { + nj.LinkTarget = string(nj.LinkTargetRaw) + nj.LinkTargetRaw = nil + } + return nil } func (node Node) Equals(other Node) bool { diff --git a/internal/restic/node_test.go b/internal/restic/node_test.go index 45ccd790c..aae010421 100644 --- a/internal/restic/node_test.go +++ b/internal/restic/node_test.go @@ -2,6 +2,8 @@ package restic_test import ( "context" + "encoding/json" + "fmt" "os" "path/filepath" "reflect" @@ -10,6 +12,7 @@ import ( "time" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test" ) @@ -334,3 +337,38 @@ func TestFixTime(t *testing.T) { }) } } + +func TestSymlinkSerialization(t *testing.T) { + for _, link := range []string{ + "válîd \t Üñi¢òde \n śẗŕinǵ", + string([]byte{0, 1, 2, 0xfa, 0xfb, 0xfc}), + } { + n := restic.Node{ + LinkTarget: link, + } + ser, err := json.Marshal(n) + test.OK(t, err) + var n2 restic.Node + err = json.Unmarshal(ser, &n2) + test.OK(t, err) + fmt.Println(string(ser)) + + test.Equals(t, n.LinkTarget, n2.LinkTarget) + } +} + +func TestSymlinkSerializationFormat(t *testing.T) { + for _, d := range []struct { + ser string + linkTarget string + }{ + {`{"linktarget":"test"}`, "test"}, + {`{"linktarget":"\u0000\u0001\u0002\ufffd\ufffd\ufffd","linktarget_raw":"AAEC+vv8"}`, string([]byte{0, 1, 2, 0xfa, 0xfb, 0xfc})}, + } { + var n2 restic.Node + err := json.Unmarshal([]byte(d.ser), &n2) + test.OK(t, err) + test.Equals(t, d.linkTarget, n2.LinkTarget) + test.Assert(t, n2.LinkTargetRaw == nil, "quoted link target is just a helper field and must be unset after decoding") + } +} diff --git a/internal/restic/parallel.go b/internal/restic/parallel.go index 34a2a019c..b22a249fe 100644 --- a/internal/restic/parallel.go +++ b/internal/restic/parallel.go @@ -8,7 +8,6 @@ import ( ) func ParallelList(ctx context.Context, r Lister, t FileType, parallelism uint, fn func(context.Context, ID, int64) error) error { - type FileIDInfo struct { ID Size int64 @@ -22,17 +21,11 @@ func ParallelList(ctx context.Context, r Lister, t FileType, parallelism uint, f // send list of index files through ch, which is closed afterwards wg.Go(func() error { defer close(ch) - return r.List(ctx, t, func(fi FileInfo) error { - id, err := ParseID(fi.Name) - if err != nil { - debug.Log("unable to parse %v as an ID", fi.Name) - return nil - } - + return r.List(ctx, t, func(id ID, size int64) error { select { case <-ctx.Done(): return nil - case ch <- FileIDInfo{id, fi.Size}: + case ch <- FileIDInfo{id, size}: } return nil }) diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 6990200e4..895c930dd 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -3,6 +3,7 @@ package restic import ( "context" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/ui/progress" @@ -17,14 +18,14 @@ var ErrInvalidData = errors.New("invalid data returned") type Repository interface { // Backend returns the backend used by the repository - Backend() Backend + Backend() backend.Backend // Connections returns the maximum number of concurrent backend operations Connections() uint Key() *crypto.Key Index() MasterIndex - LoadIndex(context.Context) error + LoadIndex(context.Context, *progress.Counter) error SetIndex(MasterIndex) error LookupBlobSize(ID, BlobType) (uint, bool) @@ -39,7 +40,7 @@ type Repository interface { List(ctx context.Context, t FileType, fn func(ID, int64) error) error // ListPack returns the list of blobs saved in the pack id and the length of - // the the pack header. + // the pack header. ListPack(context.Context, ID, int64) ([]Blob, uint32, error) LoadBlob(context.Context, BlobType, ID, []byte) ([]byte, error) @@ -56,10 +57,17 @@ type Repository interface { SaveUnpacked(context.Context, FileType, []byte) (ID, error) } -// Lister allows listing files in a backend. -type Lister interface { - List(context.Context, FileType, func(FileInfo) error) error -} +type FileType = backend.FileType + +// These are the different data types a backend can store. +const ( + PackFile FileType = backend.PackFile + KeyFile FileType = backend.KeyFile + LockFile FileType = backend.LockFile + SnapshotFile FileType = backend.SnapshotFile + IndexFile FileType = backend.IndexFile + ConfigFile FileType = backend.ConfigFile +) // LoaderUnpacked allows loading a blob not stored in a pack file type LoaderUnpacked interface { @@ -92,3 +100,8 @@ type MasterIndex interface { Save(ctx context.Context, repo SaverUnpacked, packBlacklist IDSet, extraObsolete IDs, p *progress.Counter) (obsolete IDSet, err error) } + +// Lister allows listing files in a backend. +type Lister interface { + List(ctx context.Context, t FileType, fn func(ID, int64) error) error +} diff --git a/internal/restic/snapshot_find.go b/internal/restic/snapshot_find.go index b577b0919..cb761aee3 100644 --- a/internal/restic/snapshot_find.go +++ b/internal/restic/snapshot_find.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "path/filepath" + "strings" "time" "github.com/restic/restic/internal/errors" @@ -82,37 +83,48 @@ func (f *SnapshotFilter) findLatest(ctx context.Context, be Lister, loader Loade return latest, nil } +func splitSnapshotID(s string) (id, subfolder string) { + id, subfolder, _ = strings.Cut(s, ":") + return +} + // FindSnapshot takes a string and tries to find a snapshot whose ID matches // the string as closely as possible. -func FindSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, s string) (*Snapshot, error) { +func FindSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, s string) (*Snapshot, string, error) { + s, subfolder := splitSnapshotID(s) + // no need to list snapshots if `s` is already a full id id, err := ParseID(s) if err != nil { // find snapshot id with prefix id, err = Find(ctx, be, SnapshotFile, s) if err != nil { - return nil, err + return nil, "", err } } - return LoadSnapshot(ctx, loader, id) + sn, err := LoadSnapshot(ctx, loader, id) + return sn, subfolder, err } // FindLatest returns either the latest of a filtered list of all snapshots // or a snapshot specified by `snapshotID`. -func (f *SnapshotFilter) FindLatest(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotID string) (*Snapshot, error) { - if snapshotID == "latest" { +func (f *SnapshotFilter) FindLatest(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotID string) (*Snapshot, string, error) { + id, subfolder := splitSnapshotID(snapshotID) + if id == "latest" { sn, err := f.findLatest(ctx, be, loader) if err == ErrNoSnapshotFound { err = fmt.Errorf("snapshot filter (Paths:%v Tags:%v Hosts:%v): %w", f.Paths, f.Tags, f.Hosts, err) } - return sn, err + return sn, subfolder, err } return FindSnapshot(ctx, be, loader, snapshotID) } type SnapshotFindCb func(string, *Snapshot, error) error +var ErrInvalidSnapshotSyntax = errors.New(": syntax not allowed") + // FindAll yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. func (f *SnapshotFilter) FindAll(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotIDs []string, fn SnapshotFindCb) error { if len(snapshotIDs) != 0 { @@ -138,9 +150,14 @@ func (f *SnapshotFilter) FindAll(ctx context.Context, be Lister, loader LoaderUn if sn != nil { ids.Insert(*sn.ID()) } + } else if strings.HasPrefix(s, "latest:") { + err = ErrInvalidSnapshotSyntax } else { - sn, err = FindSnapshot(ctx, be, loader, s) - if err == nil { + var subfolder string + sn, subfolder, err = FindSnapshot(ctx, be, loader, s) + if err == nil && subfolder != "" { + err = ErrInvalidSnapshotSyntax + } else if err == nil { if ids.Has(*sn.ID()) { continue } diff --git a/internal/restic/snapshot_find_test.go b/internal/restic/snapshot_find_test.go index d098b5224..84bffd694 100644 --- a/internal/restic/snapshot_find_test.go +++ b/internal/restic/snapshot_find_test.go @@ -6,16 +6,17 @@ import ( "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" ) func TestFindLatestSnapshot(t *testing.T) { repo := repository.TestRepository(t) - restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1, 0) - restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1, 0) - latestSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1, 0) + restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1) + restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1) + latestSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1) f := restic.SnapshotFilter{Hosts: []string{"foo"}} - sn, err := f.FindLatest(context.TODO(), repo.Backend(), repo, "latest") + sn, _, err := f.FindLatest(context.TODO(), repo, repo, "latest") if err != nil { t.Fatalf("FindLatest returned error: %v", err) } @@ -27,14 +28,14 @@ func TestFindLatestSnapshot(t *testing.T) { func TestFindLatestSnapshotWithMaxTimestamp(t *testing.T) { repo := repository.TestRepository(t) - restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1, 0) - desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1, 0) - restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1, 0) + restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1) + desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1) + restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1) - sn, err := (&restic.SnapshotFilter{ + sn, _, err := (&restic.SnapshotFilter{ Hosts: []string{"foo"}, TimestampLimit: parseTimeUTC("2018-08-08 08:08:08"), - }).FindLatest(context.TODO(), repo.Backend(), repo, "latest") + }).FindLatest(context.TODO(), repo, repo, "latest") if err != nil { t.Fatalf("FindLatest returned error: %v", err) } @@ -43,3 +44,48 @@ func TestFindLatestSnapshotWithMaxTimestamp(t *testing.T) { t.Errorf("FindLatest returned wrong snapshot ID: %v", *sn.ID()) } } + +func TestFindLatestWithSubpath(t *testing.T) { + repo := repository.TestRepository(t) + restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1) + desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1) + + for _, exp := range []struct { + query string + subfolder string + }{ + {"latest", ""}, + {"latest:subfolder", "subfolder"}, + {desiredSnapshot.ID().Str(), ""}, + {desiredSnapshot.ID().Str() + ":subfolder", "subfolder"}, + {desiredSnapshot.ID().String(), ""}, + {desiredSnapshot.ID().String() + ":subfolder", "subfolder"}, + } { + t.Run("", func(t *testing.T) { + sn, subfolder, err := (&restic.SnapshotFilter{}).FindLatest(context.TODO(), repo, repo, exp.query) + if err != nil { + t.Fatalf("FindLatest returned error: %v", err) + } + + test.Assert(t, *sn.ID() == *desiredSnapshot.ID(), "FindLatest returned wrong snapshot ID: %v", *sn.ID()) + test.Assert(t, subfolder == exp.subfolder, "FindLatest returned wrong path in snapshot: %v", subfolder) + }) + } +} + +func TestFindAllSubpathError(t *testing.T) { + repo := repository.TestRepository(t) + desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1) + + count := 0 + test.OK(t, (&restic.SnapshotFilter{}).FindAll(context.TODO(), repo, repo, + []string{"latest:subfolder", desiredSnapshot.ID().Str() + ":subfolder"}, + func(id string, sn *restic.Snapshot, err error) error { + if err == restic.ErrInvalidSnapshotSyntax { + count++ + return nil + } + return err + })) + test.Assert(t, count == 2, "unexpected number of subfolder errors: %v, wanted %v", count, 2) +} diff --git a/internal/restic/snapshot_group_test.go b/internal/restic/snapshot_group_test.go index 78ac99ab1..f9d6ff460 100644 --- a/internal/restic/snapshot_group_test.go +++ b/internal/restic/snapshot_group_test.go @@ -38,7 +38,7 @@ func TestGroupByOptions(t *testing.T) { var opts restic.SnapshotGroupByOptions test.OK(t, opts.Set(exp.from)) if !cmp.Equal(opts, exp.opts) { - t.Errorf("unexpeted opts %s", cmp.Diff(opts, exp.opts)) + t.Errorf("unexpected opts %s", cmp.Diff(opts, exp.opts)) } test.Equals(t, opts.String(), exp.normalized) } diff --git a/internal/restic/testdata/used_blobs_snapshot0 b/internal/restic/testdata/used_blobs_snapshot0 index 667ad34db..cc789f043 100644 --- a/internal/restic/testdata/used_blobs_snapshot0 +++ b/internal/restic/testdata/used_blobs_snapshot0 @@ -1,7 +1,8 @@ {"ID":"05bddd650a800f83f7c0d844cecb1e02f99ce962df5652a53842be50386078e1","Type":"data"} {"ID":"087040b12f129e89e4eab2b86aa14467404366a17a6082efb0d11fa7e2f9f58e","Type":"data"} +{"ID":"08a650e4d7575177ddeabf6a96896b76fa7e621aa3dd75e77293f22ce6c0c420","Type":"tree"} {"ID":"1e0f0e5799b9d711e07883050366c7eee6b7481c0d884694093149f6c4e9789a","Type":"data"} -{"ID":"229eac8e4e6c2e8d7b1d9f9627ab5d1a59cb17c5744c1e3634215116e7a92e7d","Type":"tree"} +{"ID":"435b9207cd489b41a7d119e0d75eab2a861e2b3c8d4d12ac51873ff76be0cf73","Type":"tree"} {"ID":"4719f8a039f5b745e16cf90e5b84c9255c290d500da716f7dd25909cdabb85b6","Type":"data"} {"ID":"4e352975938a29711c3003c498185972235af261a6cf8cf700a8a6ee4f914b05","Type":"data"} {"ID":"606772eacb7fe1a79267088dcadd13431914854faf1d39d47fe99a26b9fecdcb","Type":"data"} @@ -9,7 +10,6 @@ {"ID":"72b6eb0fd0d87e00392f8b91efc1a4c3f7f5c0c76f861b38aea054bc9d43463b","Type":"data"} {"ID":"77ab53b52e0cf13b300d1b7f6dac89287c8d86769d85e8a273311006ce6359be","Type":"data"} {"ID":"99dab094430d3c1be22c801a6ad7364d490a8d2ce3f9dfa3d2677431446925f4","Type":"data"} -{"ID":"9face1b278a49ef8819fbc1855ce573a85077453bbf6683488cad7767c3a38a7","Type":"tree"} {"ID":"a4c97189465344038584e76c965dd59100eaed051db1fa5ba0e143897e2c87f1","Type":"data"} {"ID":"a69c8621776ca8bb34c6c90e5ad811ddc8e2e5cfd6bb0cec5e75cca70e0b9ade","Type":"data"} {"ID":"b11f4dd9d2722b3325186f57cd13a71a3af7791118477f355b49d101104e4c22","Type":"data"} @@ -19,5 +19,5 @@ {"ID":"b9e634143719742fe77feed78b61f09573d59d2efa23d6d54afe6c159d220503","Type":"data"} {"ID":"ca896fc9ebf95fcffd7c768b07b92110b21e332a47fef7e382bf15363b0ece1a","Type":"data"} {"ID":"e6fe3512ea23a4ebf040d30958c669f7ffe724400f155a756467a9f3cafc27c5","Type":"data"} -{"ID":"e96774ac5abfbb59940939f614d65a397fb7b5abba76c29bfe14479c6616eea0","Type":"tree"} {"ID":"ed00928ce97ac5acd27c862d9097e606536e9063af1c47481257811f66260f3a","Type":"data"} +{"ID":"fb62dd9093c4958b019b90e591b2d36320ff381a24bdc9c5db3b8960ff94d174","Type":"tree"} diff --git a/internal/restic/testdata/used_blobs_snapshot1 b/internal/restic/testdata/used_blobs_snapshot1 index a5e8caedf..aa840294a 100644 --- a/internal/restic/testdata/used_blobs_snapshot1 +++ b/internal/restic/testdata/used_blobs_snapshot1 @@ -1,4 +1,3 @@ -{"ID":"04ff190aea26dae65ba4c782926cdfb700b484a8b802a5ffd58e3fadcf70b797","Type":"tree"} {"ID":"05bddd650a800f83f7c0d844cecb1e02f99ce962df5652a53842be50386078e1","Type":"data"} {"ID":"18dcaa1a676823c909aafabbb909652591915eebdde4f9a65cee955157583494","Type":"data"} {"ID":"4719f8a039f5b745e16cf90e5b84c9255c290d500da716f7dd25909cdabb85b6","Type":"data"} @@ -8,8 +7,9 @@ {"ID":"a69c8621776ca8bb34c6c90e5ad811ddc8e2e5cfd6bb0cec5e75cca70e0b9ade","Type":"data"} {"ID":"b1f2ae9d748035e5bd9a87f2579405166d150c6560d8919496f02855e1c36cf9","Type":"data"} {"ID":"b9e634143719742fe77feed78b61f09573d59d2efa23d6d54afe6c159d220503","Type":"data"} -{"ID":"bdd5a029dd295e5998c518022547d185794e72d8f8c38709a638c5841284daef","Type":"tree"} {"ID":"ca896fc9ebf95fcffd7c768b07b92110b21e332a47fef7e382bf15363b0ece1a","Type":"data"} {"ID":"cc4cab5b20a3a88995f8cdb8b0698d67a32dbc5b54487f03cb612c30a626af39","Type":"data"} {"ID":"e6fe3512ea23a4ebf040d30958c669f7ffe724400f155a756467a9f3cafc27c5","Type":"data"} +{"ID":"e9f3c4fe78e903cba60d310a9668c42232c8274b3f29b5ecebb6ff1aaeabd7e3","Type":"tree"} {"ID":"ed00928ce97ac5acd27c862d9097e606536e9063af1c47481257811f66260f3a","Type":"data"} +{"ID":"ff58f76c2313e68aa9aaaece855183855ac4ff682910404c2ae33dc999ebaca2","Type":"tree"} diff --git a/internal/restic/testdata/used_blobs_snapshot2 b/internal/restic/testdata/used_blobs_snapshot2 index f6404737e..3ed193f53 100644 --- a/internal/restic/testdata/used_blobs_snapshot2 +++ b/internal/restic/testdata/used_blobs_snapshot2 @@ -1,6 +1,7 @@ {"ID":"05bddd650a800f83f7c0d844cecb1e02f99ce962df5652a53842be50386078e1","Type":"data"} {"ID":"087040b12f129e89e4eab2b86aa14467404366a17a6082efb0d11fa7e2f9f58e","Type":"data"} {"ID":"0b88f99abc5ac71c54b3e8263c52ecb7d8903462779afdb3c8176ec5c4bb04fb","Type":"data"} +{"ID":"0e1a817fca83f569d1733b11eba14b6c9b176e41bca3644eed8b29cb907d84d3","Type":"tree"} {"ID":"1e0f0e5799b9d711e07883050366c7eee6b7481c0d884694093149f6c4e9789a","Type":"data"} {"ID":"27917462f89cecae77a4c8fb65a094b9b75a917f13794c628b1640b17f4c4981","Type":"data"} {"ID":"32745e4b26a5883ecec272c9fbfe7f3c9835c9ab41c9a2baa4d06f319697a0bd","Type":"data"} @@ -10,15 +11,14 @@ {"ID":"6b5fd3a9baf615489c82a99a71f9917bf9a2d82d5f640d7f47d175412c4b8d19","Type":"data"} {"ID":"95c97192efa810ccb1cee112238dca28673fbffce205d75ce8cc990a31005a51","Type":"data"} {"ID":"99dab094430d3c1be22c801a6ad7364d490a8d2ce3f9dfa3d2677431446925f4","Type":"data"} -{"ID":"9face1b278a49ef8819fbc1855ce573a85077453bbf6683488cad7767c3a38a7","Type":"tree"} {"ID":"a4c97189465344038584e76c965dd59100eaed051db1fa5ba0e143897e2c87f1","Type":"data"} -{"ID":"a5f2ffcd54e28e2ef3089c35b72aafda66161125e23dad581087ccd050c111c3","Type":"tree"} {"ID":"a69c8621776ca8bb34c6c90e5ad811ddc8e2e5cfd6bb0cec5e75cca70e0b9ade","Type":"data"} -{"ID":"ab5205525de94e564e3a00f634fcf9ebc397debd567734c68da7b406e612aae4","Type":"tree"} {"ID":"b6a7e8d2aa717e0a6bd68abab512c6b566074b5a6ca2edf4cd446edc5857d732","Type":"data"} -{"ID":"be2055b7125ccf824fcfa8faa4eb3985119012bac26643944eee46218e71306e","Type":"tree"} +{"ID":"bad84ed273c5fbfb40aa839a171675b7f16f5e67f3eaf4448730caa0ee27297c","Type":"tree"} {"ID":"bfc2fdb527b0c9f66bbb8d4ff1c44023cc2414efcc7f0831c10debab06bb4388","Type":"tree"} {"ID":"ca896fc9ebf95fcffd7c768b07b92110b21e332a47fef7e382bf15363b0ece1a","Type":"data"} +{"ID":"d1d3137eb08de6d8c5d9f44788c45a9fea9bb082e173bed29a0945b3347f2661","Type":"tree"} {"ID":"e6fe3512ea23a4ebf040d30958c669f7ffe724400f155a756467a9f3cafc27c5","Type":"data"} {"ID":"ed00928ce97ac5acd27c862d9097e606536e9063af1c47481257811f66260f3a","Type":"data"} {"ID":"f3cd67d9c14d2a81663d63522ab914e465b021a3b65e2f1ea6caf7478f2ec139","Type":"data"} +{"ID":"fb62dd9093c4958b019b90e591b2d36320ff381a24bdc9c5db3b8960ff94d174","Type":"tree"} diff --git a/internal/restic/testing.go b/internal/restic/testing.go index 658935eb6..d2acd3ee9 100644 --- a/internal/restic/testing.go +++ b/internal/restic/testing.go @@ -2,7 +2,6 @@ package restic import ( "context" - "encoding/json" "fmt" "io" "math/rand" @@ -19,12 +18,11 @@ func fakeFile(seed, size int64) io.Reader { } type fakeFileSystem struct { - t testing.TB - repo Repository - duplication float32 - buf []byte - chunker *chunker.Chunker - rand *rand.Rand + t testing.TB + repo Repository + buf []byte + chunker *chunker.Chunker + rand *rand.Rand } // saveFile reads from rd and saves the blobs in the repository. The list of @@ -51,13 +49,9 @@ func (fs *fakeFileSystem) saveFile(ctx context.Context, rd io.Reader) (blobs IDs fs.t.Fatalf("unable to save chunk in repo: %v", err) } - id := Hash(chunk.Data) - if !fs.blobIsKnown(BlobHandle{ID: id, Type: DataBlob}) { - _, _, _, err := fs.repo.SaveBlob(ctx, DataBlob, chunk.Data, id, true) - if err != nil { - fs.t.Fatalf("error saving chunk: %v", err) - } - + id, _, _, err := fs.repo.SaveBlob(ctx, DataBlob, chunk.Data, ID{}, false) + if err != nil { + fs.t.Fatalf("error saving chunk: %v", err) } blobs = append(blobs, id) @@ -72,30 +66,6 @@ const ( maxNodes = 15 ) -func (fs *fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) { - data, err := json.Marshal(tree) - if err != nil { - fs.t.Fatalf("json.Marshal(tree) returned error: %v", err) - return false, nil, ID{} - } - data = append(data, '\n') - - id := Hash(data) - return fs.blobIsKnown(BlobHandle{ID: id, Type: TreeBlob}), data, id -} - -func (fs *fakeFileSystem) blobIsKnown(bh BlobHandle) bool { - if fs.rand.Float32() < fs.duplication { - return false - } - - if fs.repo.Index().Has(bh) { - return true - } - - return false -} - // saveTree saves a tree of fake files in the repo and returns the ID. func (fs *fakeFileSystem) saveTree(ctx context.Context, seed int64, depth int) ID { rnd := rand.NewSource(seed) @@ -134,16 +104,12 @@ func (fs *fakeFileSystem) saveTree(ctx context.Context, seed int64, depth int) I tree.Nodes = append(tree.Nodes, node) } - known, buf, id := fs.treeIsKnown(&tree) - if known { - return id - } + tree.Sort() - _, _, _, err := fs.repo.SaveBlob(ctx, TreeBlob, buf, id, false) + id, err := SaveTree(ctx, fs.repo, &tree) if err != nil { - fs.t.Fatal(err) + fs.t.Fatalf("SaveTree returned error: %v", err) } - return id } @@ -152,22 +118,20 @@ func (fs *fakeFileSystem) saveTree(ctx context.Context, seed int64, depth int) I // also used as the snapshot's timestamp. The tree's depth can be specified // with the parameter depth. The parameter duplication is a probability that // the same blob will saved again. -func TestCreateSnapshot(t testing.TB, repo Repository, at time.Time, depth int, duplication float32) *Snapshot { +func TestCreateSnapshot(t testing.TB, repo Repository, at time.Time, depth int) *Snapshot { seed := at.Unix() t.Logf("create fake snapshot at %s with seed %d", at, seed) fakedir := fmt.Sprintf("fakedir-at-%v", at.Format("2006-01-02 15:04:05")) - snapshot, err := NewSnapshot([]string{fakedir}, []string{"test"}, "foo", time.Now()) + snapshot, err := NewSnapshot([]string{fakedir}, []string{"test"}, "foo", at) if err != nil { t.Fatal(err) } - snapshot.Time = at fs := fakeFileSystem{ - t: t, - repo: repo, - duplication: duplication, - rand: rand.New(rand.NewSource(seed)), + t: t, + repo: repo, + rand: rand.New(rand.NewSource(seed)), } var wg errgroup.Group @@ -223,3 +187,22 @@ func ParseDurationOrPanic(s string) Duration { return d } + +// TestLoadAllSnapshots returns a list of all snapshots in the repo. +// If a snapshot ID is in excludeIDs, it will not be included in the result. +func TestLoadAllSnapshots(ctx context.Context, repo Repository, excludeIDs IDSet) (snapshots Snapshots, err error) { + err = ForAllSnapshots(ctx, repo, repo, excludeIDs, func(id ID, sn *Snapshot, err error) error { + if err != nil { + return err + } + + snapshots = append(snapshots, sn) + return nil + }) + + if err != nil { + return nil, err + } + + return snapshots, nil +} diff --git a/internal/restic/testing_test.go b/internal/restic/testing_test.go index 2af5c607e..ae8f8dd34 100644 --- a/internal/restic/testing_test.go +++ b/internal/restic/testing_test.go @@ -17,32 +17,13 @@ const ( testDepth = 2 ) -// LoadAllSnapshots returns a list of all snapshots in the repo. -// If a snapshot ID is in excludeIDs, it will not be included in the result. -func loadAllSnapshots(ctx context.Context, repo restic.Repository, excludeIDs restic.IDSet) (snapshots restic.Snapshots, err error) { - err = restic.ForAllSnapshots(ctx, repo.Backend(), repo, excludeIDs, func(id restic.ID, sn *restic.Snapshot, err error) error { - if err != nil { - return err - } - - snapshots = append(snapshots, sn) - return nil - }) - - if err != nil { - return nil, err - } - - return snapshots, nil -} - func TestCreateSnapshot(t *testing.T) { repo := repository.TestRepository(t) for i := 0; i < testCreateSnapshots; i++ { - restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth, 0) + restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth) } - snapshots, err := loadAllSnapshots(context.TODO(), repo, restic.NewIDSet()) + snapshots, err := restic.TestLoadAllSnapshots(context.TODO(), repo, restic.NewIDSet()) if err != nil { t.Fatal(err) } @@ -73,6 +54,6 @@ func BenchmarkTestCreateSnapshot(t *testing.B) { t.ResetTimer() for i := 0; i < t.N; i++ { - restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth, 0) + restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth) } } diff --git a/internal/restic/tree.go b/internal/restic/tree.go index 373b36746..3c3e3ab56 100644 --- a/internal/restic/tree.go +++ b/internal/restic/tree.go @@ -5,7 +5,9 @@ import ( "context" "encoding/json" "fmt" + "path" "sort" + "strings" "github.com/restic/restic/internal/errors" @@ -184,3 +186,32 @@ func (builder *TreeJSONBuilder) Finalize() ([]byte, error) { builder.buf = bytes.Buffer{} return buf, nil } + +func FindTreeDirectory(ctx context.Context, repo BlobLoader, id *ID, dir string) (*ID, error) { + if id == nil { + return nil, errors.New("tree id is null") + } + + dirs := strings.Split(path.Clean(dir), "/") + subfolder := "" + + for _, name := range dirs { + if name == "" || name == "." { + continue + } + subfolder = path.Join(subfolder, name) + tree, err := LoadTree(ctx, repo, *id) + if err != nil { + return nil, fmt.Errorf("path %s: %w", subfolder, err) + } + node := tree.Find(name) + if node == nil { + return nil, fmt.Errorf("path %s: not found", subfolder) + } + if node.Type != "dir" || node.Subtree == nil { + return nil, fmt.Errorf("path %s: not a directory", subfolder) + } + id = node.Subtree + } + return id, nil +} diff --git a/internal/restic/tree_test.go b/internal/restic/tree_test.go index fb25ca373..da674eb1c 100644 --- a/internal/restic/tree_test.go +++ b/internal/restic/tree_test.go @@ -210,3 +210,37 @@ func benchmarkLoadTree(t *testing.B, version uint) { rtest.OK(t, err) } } + +func TestFindTreeDirectory(t *testing.T) { + repo := repository.TestRepository(t) + sn := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:08"), 3) + + for _, exp := range []struct { + subfolder string + id restic.ID + err error + }{ + {"", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil}, + {"/", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil}, + {".", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil}, + {"..", restic.ID{}, errors.New("path ..: not found")}, + {"file-1", restic.ID{}, errors.New("path file-1: not a directory")}, + {"dir-21", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil}, + {"/dir-21", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil}, + {"dir-21/", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil}, + {"dir-21/dir-24", restic.TestParseID("74626b3fb2bd4b3e572b81a4059b3e912bcf2a8f69fecd9c187613b7173f13b1"), nil}, + } { + t.Run("", func(t *testing.T) { + id, err := restic.FindTreeDirectory(context.TODO(), repo, sn.Tree, exp.subfolder) + if exp.err == nil { + rtest.OK(t, err) + rtest.Assert(t, exp.id == *id, "unexpected id, expected %v, got %v", exp.id, id) + } else { + rtest.Assert(t, exp.err.Error() == err.Error(), "unexpected err, expected %v, got %v", exp.err, err) + } + }) + } + + _, err := restic.FindTreeDirectory(context.TODO(), repo, nil, "") + rtest.Assert(t, err != nil, "missing error on null tree id") +} diff --git a/internal/restorer/doc.go b/internal/restorer/doc.go index e230f23f0..8d68d7161 100644 --- a/internal/restorer/doc.go +++ b/internal/restorer/doc.go @@ -18,7 +18,7 @@ // // Implementation does not guarantee order in which blobs are written to the // target files and, for example, the last blob of a file can be written to the -// file before any of the preceding file blobs. It is therefore possible to +// file before any of the preceeding file blobs. It is therefore possible to // have gaps in the data written to the target files if restore fails or // interrupted by the user. package restorer diff --git a/internal/restorer/filerestorer_test.go b/internal/restorer/filerestorer_test.go index e798f2b8b..aa9a2392d 100644 --- a/internal/restorer/filerestorer_test.go +++ b/internal/restorer/filerestorer_test.go @@ -8,6 +8,7 @@ import ( "os" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" @@ -135,7 +136,7 @@ func newTestRepo(content []TestFile) *TestRepo { files: files, filesPathToContent: filesPathToContent, } - repo.loader = func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + repo.loader = func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { packID, err := restic.ParseID(h.Name) if err != nil { return err @@ -261,7 +262,7 @@ func TestErrorRestoreFiles(t *testing.T) { loadError := errors.New("load error") // loader always returns an error - repo.loader = func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + repo.loader = func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { return loadError } @@ -294,8 +295,8 @@ func testPartialDownloadError(t *testing.T, part int) { // loader always returns an error loader := repo.loader - repo.loader = func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - // only load partial data to execise fault handling in different places + repo.loader = func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + // only load partial data to exercise fault handling in different places err := loader(ctx, h, length*part/100, offset, fn) if err == nil { return nil diff --git a/internal/restorer/hardlinks_index.go b/internal/restorer/hardlinks_index.go index 9cf45975a..d069fb4cb 100644 --- a/internal/restorer/hardlinks_index.go +++ b/internal/restorer/hardlinks_index.go @@ -10,20 +10,20 @@ type HardlinkKey struct { } // HardlinkIndex contains a list of inodes, devices these inodes are one, and associated file names. -type HardlinkIndex struct { +type HardlinkIndex[T any] struct { m sync.Mutex - Index map[HardlinkKey]string + Index map[HardlinkKey]T } // NewHardlinkIndex create a new index for hard links -func NewHardlinkIndex() *HardlinkIndex { - return &HardlinkIndex{ - Index: make(map[HardlinkKey]string), +func NewHardlinkIndex[T any]() *HardlinkIndex[T] { + return &HardlinkIndex[T]{ + Index: make(map[HardlinkKey]T), } } -// Has checks wether the link already exist in the index. -func (idx *HardlinkIndex) Has(inode uint64, device uint64) bool { +// Has checks whether the link already exist in the index. +func (idx *HardlinkIndex[T]) Has(inode uint64, device uint64) bool { idx.m.Lock() defer idx.m.Unlock() _, ok := idx.Index[HardlinkKey{inode, device}] @@ -32,25 +32,25 @@ func (idx *HardlinkIndex) Has(inode uint64, device uint64) bool { } // Add adds a link to the index. -func (idx *HardlinkIndex) Add(inode uint64, device uint64, name string) { +func (idx *HardlinkIndex[T]) Add(inode uint64, device uint64, value T) { idx.m.Lock() defer idx.m.Unlock() _, ok := idx.Index[HardlinkKey{inode, device}] if !ok { - idx.Index[HardlinkKey{inode, device}] = name + idx.Index[HardlinkKey{inode, device}] = value } } -// GetFilename obtains the filename from the index. -func (idx *HardlinkIndex) GetFilename(inode uint64, device uint64) string { +// Value obtains the filename from the index. +func (idx *HardlinkIndex[T]) Value(inode uint64, device uint64) T { idx.m.Lock() defer idx.m.Unlock() return idx.Index[HardlinkKey{inode, device}] } // Remove removes a link from the index. -func (idx *HardlinkIndex) Remove(inode uint64, device uint64) { +func (idx *HardlinkIndex[T]) Remove(inode uint64, device uint64) { idx.m.Lock() defer idx.m.Unlock() delete(idx.Index, HardlinkKey{inode, device}) diff --git a/internal/restorer/hardlinks_index_test.go b/internal/restorer/hardlinks_index_test.go index 75a2b83ee..31ce938f9 100644 --- a/internal/restorer/hardlinks_index_test.go +++ b/internal/restorer/hardlinks_index_test.go @@ -10,15 +10,15 @@ import ( // TestHardLinks contains various tests for HardlinkIndex. func TestHardLinks(t *testing.T) { - idx := restorer.NewHardlinkIndex() + idx := restorer.NewHardlinkIndex[string]() idx.Add(1, 2, "inode1-file1-on-device2") idx.Add(2, 3, "inode2-file2-on-device3") - sresult := idx.GetFilename(1, 2) + sresult := idx.Value(1, 2) rtest.Equals(t, sresult, "inode1-file1-on-device2") - sresult = idx.GetFilename(2, 3) + sresult = idx.Value(2, 3) rtest.Equals(t, sresult, "inode2-file2-on-device3") bresult := idx.Has(1, 2) diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 3c60aca1b..e973316c0 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -230,7 +230,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { } } - idx := NewHardlinkIndex() + idx := NewHardlinkIndex[string]() filerestorer := newFileRestorer(dst, res.repo.Backend().Load, res.repo.Key(), res.repo.Index().Lookup, res.repo.Connections(), res.sparse, res.progress) filerestorer.Error = res.Error @@ -319,8 +319,8 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { return res.restoreEmptyFileAt(node, target, location) } - if idx.Has(node.Inode, node.DeviceID) && idx.GetFilename(node.Inode, node.DeviceID) != location { - return res.restoreHardlinkAt(node, filerestorer.targetPath(idx.GetFilename(node.Inode, node.DeviceID)), target, location) + if idx.Has(node.Inode, node.DeviceID) && idx.Value(node.Inode, node.DeviceID) != location { + return res.restoreHardlinkAt(node, filerestorer.targetPath(idx.Value(node.Inode, node.DeviceID)), target, location) } return res.restoreNodeMetadataTo(node, target, location) diff --git a/internal/restorer/restorer_test.go b/internal/restorer/restorer_test.go index 6c45d5556..d0e7dad6f 100644 --- a/internal/restorer/restorer_test.go +++ b/internal/restorer/restorer_test.go @@ -791,7 +791,7 @@ func TestRestorerConsistentTimestampsAndPermissions(t *testing.T) { } } -// VerifyFiles must not report cancelation of its context through res.Error. +// VerifyFiles must not report cancellation of its context through res.Error. func TestVerifyCancel(t *testing.T) { snapshot := Snapshot{ Nodes: map[string]Node{ diff --git a/internal/ui/restore/text.go b/internal/ui/restore/text.go index e6465eed0..2647bb28b 100644 --- a/internal/ui/restore/text.go +++ b/internal/ui/restore/text.go @@ -22,7 +22,7 @@ func (t *textPrinter) Update(filesFinished, filesTotal, allBytesWritten, allByte formattedAllBytesWritten := ui.FormatBytes(allBytesWritten) formattedAllBytesTotal := ui.FormatBytes(allBytesTotal) allPercent := ui.FormatPercent(allBytesWritten, allBytesTotal) - progress := fmt.Sprintf("[%s] %s %v files %s, total %v files %v", + progress := fmt.Sprintf("[%s] %s %v files/dirs %s, total %v files/dirs %v", timeLeft, allPercent, filesFinished, formattedAllBytesWritten, filesTotal, formattedAllBytesTotal) t.terminal.SetStatus([]string{progress}) @@ -36,10 +36,10 @@ func (t *textPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allByte var summary string if filesFinished == filesTotal && allBytesWritten == allBytesTotal { - summary = fmt.Sprintf("Summary: Restored %d Files (%s) in %s", filesTotal, formattedAllBytesTotal, timeLeft) + summary = fmt.Sprintf("Summary: Restored %d files/dirs (%s) in %s", filesTotal, formattedAllBytesTotal, timeLeft) } else { formattedAllBytesWritten := ui.FormatBytes(allBytesWritten) - summary = fmt.Sprintf("Summary: Restored %d / %d Files (%s / %s) in %s", + summary = fmt.Sprintf("Summary: Restored %d / %d files/dirs (%s / %s) in %s", filesFinished, filesTotal, formattedAllBytesWritten, formattedAllBytesTotal, timeLeft) } diff --git a/internal/ui/restore/text_test.go b/internal/ui/restore/text_test.go index 2a8c90878..fc03904ff 100644 --- a/internal/ui/restore/text_test.go +++ b/internal/ui/restore/text_test.go @@ -23,19 +23,19 @@ func TestPrintUpdate(t *testing.T) { term := &mockTerm{} printer := NewTextProgress(term) printer.Update(3, 11, 29, 47, 5*time.Second) - test.Equals(t, []string{"[0:05] 61.70% 3 files 29 B, total 11 files 47 B"}, term.output) + test.Equals(t, []string{"[0:05] 61.70% 3 files/dirs 29 B, total 11 files/dirs 47 B"}, term.output) } func TestPrintSummaryOnSuccess(t *testing.T) { term := &mockTerm{} printer := NewTextProgress(term) printer.Finish(11, 11, 47, 47, 5*time.Second) - test.Equals(t, []string{"Summary: Restored 11 Files (47 B) in 0:05"}, term.output) + test.Equals(t, []string{"Summary: Restored 11 files/dirs (47 B) in 0:05"}, term.output) } func TestPrintSummaryOnErrors(t *testing.T) { term := &mockTerm{} printer := NewTextProgress(term) printer.Finish(3, 11, 29, 47, 5*time.Second) - test.Equals(t, []string{"Summary: Restored 3 / 11 Files (29 B / 47 B) in 0:05"}, term.output) + test.Equals(t, []string{"Summary: Restored 3 / 11 files/dirs (29 B / 47 B) in 0:05"}, term.output) } diff --git a/internal/ui/termstatus/status.go b/internal/ui/termstatus/status.go index 6e8ddfe7c..95286de99 100644 --- a/internal/ui/termstatus/status.go +++ b/internal/ui/termstatus/status.go @@ -325,7 +325,7 @@ func Truncate(s string, w int) string { // Guess whether the first rune in s would occupy two terminal cells // instead of one. This cannot be determined exactly without knowing -// the terminal font, so we treat all ambigous runes as full-width, +// the terminal font, so we treat all ambiguous runes as full-width, // i.e., two cells. func wideRune(s string) (wide bool, utfsize uint) { prop, size := width.LookupString(s) diff --git a/internal/walker/rewriter_test.go b/internal/walker/rewriter_test.go index 716217ac6..e5fcb9915 100644 --- a/internal/walker/rewriter_test.go +++ b/internal/walker/rewriter_test.go @@ -69,7 +69,7 @@ func checkRewriteItemOrder(want []string) checkRewriteFunc { } } -// checkRewriteSkips excludes nodes if path is in skipFor, it checks that rewriting proceedes in the correct order. +// checkRewriteSkips excludes nodes if path is in skipFor, it checks that rewriting proceeds in the correct order. func checkRewriteSkips(skipFor map[string]struct{}, want []string, disableCache bool) checkRewriteFunc { var pos int