mirror of
https://github.com/restic/restic.git
synced 2025-03-09 00:00:02 +01:00

* feat(backends/s3): add warmup support before repacks and restores This commit introduces basic support for transitioning pack files stored in cold storage to hot storage on S3 and S3-compatible providers. To prevent unexpected behavior for existing users, the feature is gated behind new flags: - `s3.enable-restore`: opt-in flag (defaults to false) - `s3.restore-days`: number of days for the restored objects to remain in hot storage (defaults to `7`) - `s3.restore-timeout`: maximum time to wait for a single restoration (default to `1 day`) - `s3.restore-tier`: retrieval tier at which the restore will be processed. (default to `Standard`) As restoration times can be lengthy, this implementation preemptively restores selected packs to prevent incessant restore-delays during downloads. This is slightly sub-optimal as we could process packs out-of-order (as soon as they're transitioned), but this would really add too much complexity for a marginal gain in speed. To maintain simplicity and prevent resources exhautions with lots of packs, no new concurrency mechanisms or goroutines were added. This just hooks gracefully into the existing routines. **Limitations:** - Tests against the backend were not written due to the lack of cold storage class support in MinIO. Testing was done manually on Scaleway's S3-compatible object storage. If necessary, we could explore testing with LocalStack or mocks, though this requires further discussion. - Currently, this feature only warms up before restores and repacks (prune/copy), as those are the two main use-cases I came across. Support for other commands may be added in future iterations, as long as affected packs can be calculated in advance. - The feature is gated behind a new alpha `s3-restore` feature flag to make it explicit that the feature is still wet behind the ears. - There is no explicit user notification for ongoing pack restorations. While I think it is not necessary because of the opt-in flag, showing some notice may improve usability (but would probably require major refactoring in the progress bar which I didn't want to start). Another possibility would be to add a flag to send restores requests and fail early. See https://github.com/restic/restic/issues/3202 * ui: warn user when files are warming up from cold storage * refactor: remove the PacksWarmer struct It's easier to handle multiple handles in the backend directly, and it may open the door to reducing the number of requests made to the backend in the future.
172 lines
4.3 KiB
Go
172 lines
4.3 KiB
Go
package mock
|
|
|
|
import (
|
|
"context"
|
|
"hash"
|
|
"io"
|
|
|
|
"github.com/restic/restic/internal/backend"
|
|
"github.com/restic/restic/internal/errors"
|
|
)
|
|
|
|
// Backend implements a mock backend.
|
|
type Backend struct {
|
|
CloseFn func() error
|
|
IsNotExistFn func(err error) bool
|
|
IsPermanentErrorFn func(err error) bool
|
|
SaveFn func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error
|
|
OpenReaderFn func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error)
|
|
StatFn func(ctx context.Context, h backend.Handle) (backend.FileInfo, error)
|
|
ListFn func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error
|
|
RemoveFn func(ctx context.Context, h backend.Handle) error
|
|
DeleteFn func(ctx context.Context) error
|
|
WarmupFn func(ctx context.Context, h []backend.Handle) ([]backend.Handle, error)
|
|
WarmupWaitFn func(ctx context.Context, h []backend.Handle) error
|
|
ConnectionsFn func() uint
|
|
HasherFn func() hash.Hash
|
|
HasAtomicReplaceFn func() bool
|
|
}
|
|
|
|
// NewBackend returns new mock Backend instance
|
|
func NewBackend() *Backend {
|
|
be := &Backend{}
|
|
return be
|
|
}
|
|
|
|
// Close the backend.
|
|
func (m *Backend) Close() error {
|
|
if m.CloseFn == nil {
|
|
return nil
|
|
}
|
|
|
|
return m.CloseFn()
|
|
}
|
|
|
|
func (m *Backend) Connections() uint {
|
|
if m.ConnectionsFn == nil {
|
|
return 2
|
|
}
|
|
|
|
return m.ConnectionsFn()
|
|
}
|
|
|
|
// Hasher may return a hash function for calculating a content hash for the backend
|
|
func (m *Backend) Hasher() hash.Hash {
|
|
if m.HasherFn == nil {
|
|
return nil
|
|
}
|
|
|
|
return m.HasherFn()
|
|
}
|
|
|
|
// HasAtomicReplace returns whether Save() can atomically replace files
|
|
func (m *Backend) HasAtomicReplace() bool {
|
|
if m.HasAtomicReplaceFn == nil {
|
|
return false
|
|
}
|
|
return m.HasAtomicReplaceFn()
|
|
}
|
|
|
|
// IsNotExist returns true if the error is caused by a missing file.
|
|
func (m *Backend) IsNotExist(err error) bool {
|
|
if m.IsNotExistFn == nil {
|
|
return false
|
|
}
|
|
|
|
return m.IsNotExistFn(err)
|
|
}
|
|
|
|
func (m *Backend) IsPermanentError(err error) bool {
|
|
if m.IsPermanentErrorFn == nil {
|
|
return false
|
|
}
|
|
|
|
return m.IsPermanentErrorFn(err)
|
|
}
|
|
|
|
// Save data in the backend.
|
|
func (m *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
|
|
if m.SaveFn == nil {
|
|
return errors.New("not implemented")
|
|
}
|
|
|
|
return m.SaveFn(ctx, h, rd)
|
|
}
|
|
|
|
// Load runs fn with a reader that yields the contents of the file at h at the
|
|
// given offset.
|
|
func (m *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
|
|
rd, err := m.openReader(ctx, h, length, offset)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = fn(rd)
|
|
if err != nil {
|
|
_ = rd.Close() // ignore secondary errors closing the reader
|
|
return err
|
|
}
|
|
return rd.Close()
|
|
}
|
|
|
|
func (m *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
|
|
if m.OpenReaderFn == nil {
|
|
return nil, errors.New("not implemented")
|
|
}
|
|
|
|
return m.OpenReaderFn(ctx, h, length, offset)
|
|
}
|
|
|
|
// Stat an object in the backend.
|
|
func (m *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
|
|
if m.StatFn == nil {
|
|
return backend.FileInfo{}, errors.New("not implemented")
|
|
}
|
|
|
|
return m.StatFn(ctx, h)
|
|
}
|
|
|
|
// List items of type t.
|
|
func (m *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
|
|
if m.ListFn == nil {
|
|
return nil
|
|
}
|
|
|
|
return m.ListFn(ctx, t, fn)
|
|
}
|
|
|
|
// Remove data from the backend.
|
|
func (m *Backend) Remove(ctx context.Context, h backend.Handle) error {
|
|
if m.RemoveFn == nil {
|
|
return errors.New("not implemented")
|
|
}
|
|
|
|
return m.RemoveFn(ctx, h)
|
|
}
|
|
|
|
// Delete all data.
|
|
func (m *Backend) Delete(ctx context.Context) error {
|
|
if m.DeleteFn == nil {
|
|
return errors.New("not implemented")
|
|
}
|
|
|
|
return m.DeleteFn(ctx)
|
|
}
|
|
|
|
func (m *Backend) Warmup(ctx context.Context, h []backend.Handle) ([]backend.Handle, error) {
|
|
if m.WarmupFn == nil {
|
|
return []backend.Handle{}, errors.New("not implemented")
|
|
}
|
|
|
|
return m.WarmupFn(ctx, h)
|
|
}
|
|
|
|
func (m *Backend) WarmupWait(ctx context.Context, h []backend.Handle) error {
|
|
if m.WarmupWaitFn == nil {
|
|
return errors.New("not implemented")
|
|
}
|
|
|
|
return m.WarmupWaitFn(ctx, h)
|
|
}
|
|
|
|
// Make sure that Backend implements the backend interface.
|
|
var _ backend.Backend = &Backend{}
|