Repository: restic/restic
Pull request #5296 (2025-03-23 17:11:15)

recover: reindex before reassembling snapshot
<!--
Thank you very much for contributing code or documentation to restic! Please
fill out the following questions to make it easier for us to review your
changes.
-->

What does this PR change? What problem does it solve?
-----------------------------------------------------
Run `repair index` before trying to recover leftover trees into a snapshot.

<!--
Describe the changes and their purpose here, as detailed as needed.
-->

Was the change previously discussed in an issue or on the forum?
----------------------------------------------------------------
Fixes https://github.com/restic/restic/issues/5287 .
<!--
Link issues and relevant forum posts here.

If this PR resolves an issue on GitHub, use "Closes #1234" so that the issue
is closed automatically when this PR is merged.
-->

Checklist
---------

<!--
You do not need to check all the boxes below all at once. Feel free to take
your time and add more commits. If you're done and ready for review, please
check the last box. Enable a checkbox by replacing [ ] with [x].

Please always follow these steps:
- Read the [contribution guidelines](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#providing-patches).
- Enable [maintainer edits](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork).
- Run `gofmt` on the code in all commits.
- Format all commit messages in the same style as [the other commits in the repository](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits).
-->

- [x] I have added tests for all code changes.
- [ ] I have added documentation for relevant changes (in the manual).
- [x] There's a new file in `changelog/unreleased/` that describes the changes for our users (see [template](https://github.com/restic/restic/blob/master/changelog/TEMPLATE)).
- [x] I'm done! This pull request is ready for review.



Patch:
diff --git a/changelog/unreleased/issue-5287 b/changelog/unreleased/issue-5287
new file mode 100644
index 000000000..a77876038
--- /dev/null
+++ b/changelog/unreleased/issue-5287
@@ -0,0 +1,8 @@
+Enhancement: `recover` automatically runs `repair index`
+
+When trying to recover data from an interrupted snapshot, it was necessary
+to manually run `restic repair index` before runnning `restic recover`.
+This now happens automatically.
+
+https://github.com/restic/restic/issues/52897
+https://github.com/restic/restic/pull/5296
diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go
index f87303933..f1c0516c2 100644
--- a/cmd/restic/cmd_check.go
+++ b/cmd/restic/cmd_check.go
@@ -262,10 +262,10 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	for _, hint := range hints {
 		switch hint.(type) {
 		case *checker.ErrDuplicatePacks:
-			term.Print(hint.Error())
+			printer.S("%s", hint.Error())
 			summary.HintRepairIndex = true
 		case *checker.ErrMixedPack:
-			term.Print(hint.Error())
+			printer.S("%s", hint.Error())
 			summary.HintPrune = true
 		default:
 			printer.E("error: %v\n", hint)
@@ -274,10 +274,10 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	}
 
 	if summary.HintRepairIndex {
-		term.Print("Duplicate packs are non-critical, you can run `restic repair index' to correct this.\n")
+		printer.S("Duplicate packs are non-critical, you can run `restic repair index' to correct this.\n")
 	}
 	if summary.HintPrune {
-		term.Print("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
+		printer.S("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
 	}
 
 	if len(errs) > 0 {
@@ -534,6 +534,7 @@ func (p *jsonErrorPrinter) E(msg string, args ...interface{}) {
 	}
 	p.term.Error(ui.ToJSONString(status))
 }
+func (*jsonErrorPrinter) S(_ string, _ ...interface{})  {}
 func (*jsonErrorPrinter) P(_ string, _ ...interface{})  {}
 func (*jsonErrorPrinter) V(_ string, _ ...interface{})  {}
 func (*jsonErrorPrinter) VV(_ string, _ ...interface{}) {}
diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go
index 10284c111..c1c71333f 100644
--- a/cmd/restic/cmd_recover.go
+++ b/cmd/restic/cmd_recover.go
@@ -6,7 +6,10 @@ import (
 	"time"
 
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui/progress"
+	"github.com/restic/restic/internal/ui/termstatus"
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/errgroup"
 )
@@ -32,31 +35,41 @@ Exit status is 12 if the password is incorrect.
 		GroupID:           cmdGroupDefault,
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			return runRecover(cmd.Context(), globalOptions)
+			term, cancel := setupTermstatus()
+			defer cancel()
+			return runRecover(cmd.Context(), globalOptions, term)
 		},
 	}
 	return cmd
 }
 
-func runRecover(ctx context.Context, gopts GlobalOptions) error {
+func runRecover(ctx context.Context, gopts GlobalOptions, term *termstatus.Terminal) error {
 	hostname, err := os.Hostname()
 	if err != nil {
 		return err
 	}
 
-	ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, false)
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
 	if err != nil {
 		return err
 	}
 	defer unlock()
 
+	printer := newTerminalProgressPrinter(gopts.verbosity, term)
+
 	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
 
-	Verbosef("load index files\n")
-	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	printer.P("ensuring index is complete\n")
+	err = repository.RepairIndex(ctx, repo, repository.RepairIndexOptions{}, printer)
+	if err != nil {
+		return err
+	}
+
+	printer.P("load index files\n")
+	bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
 	if err = repo.LoadIndex(ctx, bar); err != nil {
 		return err
 	}
@@ -74,15 +87,15 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 		return err
 	}
 
-	Verbosef("load %d trees\n", len(trees))
-	bar = newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
+	printer.P("load %d trees\n", len(trees))
+	bar = newTerminalProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded", term)
 	for id := range trees {
 		tree, err := restic.LoadTree(ctx, repo, id)
 		if ctx.Err() != nil {
 			return ctx.Err()
 		}
 		if err != nil {
-			Warnf("unable to load tree %v: %v\n", id.Str(), err)
+			printer.E("unable to load tree %v: %v\n", id.Str(), err)
 			continue
 		}
 
@@ -95,7 +108,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 	}
 	bar.Done()
 
-	Verbosef("load snapshots\n")
+	printer.P("load snapshots\n")
 	err = restic.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(_ restic.ID, sn *restic.Snapshot, _ error) error {
 		trees[*sn.Tree] = true
 		return nil
@@ -103,19 +116,19 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 	if err != nil {
 		return err
 	}
-	Verbosef("done\n")
+	printer.P("done\n")
 
 	roots := restic.NewIDSet()
 	for id, seen := range trees {
 		if !seen {
-			Verboseff("found root tree %v\n", id.Str())
+			printer.V("found root tree %v\n", id.Str())
 			roots.Insert(id)
 		}
 	}
-	Printf("\nfound %d unreferenced roots\n", len(roots))
+	printer.S("\nfound %d unreferenced roots\n", len(roots))
 
 	if len(roots) == 0 {
-		Verbosef("no snapshot to write.\n")
+		printer.P("no snapshot to write.\n")
 		return nil
 	}
 
@@ -163,11 +176,11 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 		return err
 	}
 
-	return createSnapshot(ctx, "/recover", hostname, []string{"recovered"}, repo, &treeID)
+	return createSnapshot(ctx, printer, "/recover", hostname, []string{"recovered"}, repo, &treeID)
 
 }
 
-func createSnapshot(ctx context.Context, name, hostname string, tags []string, repo restic.SaverUnpacked[restic.WriteableFileType], tree *restic.ID) error {
+func createSnapshot(ctx context.Context, printer progress.Printer, name, hostname string, tags []string, repo restic.SaverUnpacked[restic.WriteableFileType], tree *restic.ID) error {
 	sn, err := restic.NewSnapshot([]string{name}, tags, hostname, time.Now())
 	if err != nil {
 		return errors.Fatalf("unable to save snapshot: %v", err)
@@ -180,6 +193,6 @@ func createSnapshot(ctx context.Context, name, hostname string, tags []string, r
 		return errors.Fatalf("unable to save snapshot: %v", err)
 	}
 
-	Printf("saved new snapshot %v\n", id.Str())
+	printer.S("saved new snapshot %v\n", id.Str())
 	return nil
 }
diff --git a/go.mod b/go.mod
index 153b56159..f94924f4d 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
 	github.com/google/go-cmp v0.6.0
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/klauspost/compress v1.18.0
-	github.com/minio/minio-go/v7 v7.0.88
+	github.com/minio/minio-go/v7 v7.0.77
 	github.com/ncw/swift/v2 v2.0.3
 	github.com/peterbourgon/unixtransport v0.0.4
 	github.com/pkg/errors v0.9.1
@@ -58,7 +58,7 @@ require (
 	github.com/go-ini/ini v1.67.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/goccy/go-json v0.10.5 // indirect
+	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect
@@ -67,10 +67,9 @@ require (
 	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
 	github.com/googleapis/gax-go/v2 v2.14.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/minio/crc64nvme v1.0.1 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/rs/xid v1.6.0 // indirect
diff --git a/go.sum b/go.sum
index d428914d4..2f0ca42c6 100644
--- a/go.sum
+++ b/go.sum
@@ -86,8 +86,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
 github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
-github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
-github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
+github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
+github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
 github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -139,8 +139,8 @@ github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeW
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
-github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
+github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
+github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -154,12 +154,10 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
-github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY=
-github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v7 v7.0.88 h1:v8MoIJjwYxOkehp+eiLIuvXk87P2raUtoU5klrAAshs=
-github.com/minio/minio-go/v7 v7.0.88/go.mod h1:33+O8h0tO7pCeCWwBVa07RhVVfB/3vS4kEX7rwYKmIg=
+github.com/minio/minio-go/v7 v7.0.77 h1:GaGghJRg9nwDVlNbwYjSDJT1rqltQkBFDsypWX1v3Bw=
+github.com/minio/minio-go/v7 v7.0.77/go.mod h1:AVM3IUN6WwKzmwBxVdjzhH8xq+f57JSbbvzqvUzR6eg=
 github.com/ncw/swift/v2 v2.0.3 h1:8R9dmgFIWs+RiVlisCEfiQiik1hjuR0JnOkLxaP9ihg=
 github.com/ncw/swift/v2 v2.0.3/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk=
 github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go
index 3653c827c..2eded428c 100644
--- a/internal/backend/s3/s3.go
+++ b/internal/backend/s3/s3.go
@@ -131,11 +131,14 @@ func getCredentials(cfg Config, tr http.RoundTripper) (*credentials.Credentials,
 		&credentials.EnvMinio{},
 		&credentials.FileAWSCredentials{},
 		&credentials.FileMinioClient{},
-		&credentials.IAM{},
+		&credentials.IAM{
+			Client: &http.Client{
+				Transport: tr,
+			},
+		},
 	})
-	client := &http.Client{Transport: tr}
 
-	c, err := creds.GetWithContext(&credentials.CredContext{Client: client})
+	c, err := creds.Get()
 	if err != nil {
 		return nil, errors.Wrap(err, "creds.Get")
 	}
diff --git a/internal/ui/message.go b/internal/ui/message.go
index 6ad5a439e..6ba60f3c0 100644
--- a/internal/ui/message.go
+++ b/internal/ui/message.go
@@ -24,6 +24,12 @@ func (m *Message) E(msg string, args ...interface{}) {
 	m.term.Error(fmt.Sprintf(msg, args...))
 }
 
+// S prints a message, this is should only be used for very important messages
+// that are not errors.
+func (m *Message) S(msg string, args ...interface{}) {
+	m.term.Print(fmt.Sprintf(msg, args...))
+}
+
 // P prints a message if verbosity >= 1, this is used for normal messages which
 // are not errors.
 func (m *Message) P(msg string, args ...interface{}) {
diff --git a/internal/ui/progress/printer.go b/internal/ui/progress/printer.go
index a2bc4c4b5..c3e0d17a3 100644
--- a/internal/ui/progress/printer.go
+++ b/internal/ui/progress/printer.go
@@ -8,9 +8,15 @@ import "testing"
 type Printer interface {
 	NewCounter(description string) *Counter
 
+	// E prints to stderr
 	E(msg string, args ...interface{})
+	// S prints to stdout
+	S(msg string, args ...interface{})
+	// P prints to stdout unless quiet was passed
 	P(msg string, args ...interface{})
+	// V prints to stdout if verbose is set once
 	V(msg string, args ...interface{})
+	// VV prints to stdout if verbose is set twice
 	VV(msg string, args ...interface{})
 }
 
@@ -25,6 +31,8 @@ func (*NoopPrinter) NewCounter(_ string) *Counter {
 
 func (*NoopPrinter) E(_ string, _ ...interface{}) {}
 
+func (*NoopPrinter) S(_ string, _ ...interface{}) {}
+
 func (*NoopPrinter) P(_ string, _ ...interface{}) {}
 
 func (*NoopPrinter) V(_ string, _ ...interface{}) {}
@@ -52,6 +60,10 @@ func (p *TestPrinter) E(msg string, args ...interface{}) {
 	p.t.Logf("error: "+msg, args...)
 }
 
+func (p *TestPrinter) S(msg string, args ...interface{}) {
+	p.t.Logf("stdout: "+msg, args...)
+}
+
 func (p *TestPrinter) P(msg string, args ...interface{}) {
 	p.t.Logf("print: "+msg, args...)
 }


Test patch:
diff --git a/cmd/restic/cmd_recover_integration_test.go b/cmd/restic/cmd_recover_integration_test.go
new file mode 100644
index 000000000..c3d210200
--- /dev/null
+++ b/cmd/restic/cmd_recover_integration_test.go
@@ -0,0 +1,37 @@
+package main
+
+import (
+	"context"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+	"github.com/restic/restic/internal/ui/termstatus"
+)
+
+func testRunRecover(t testing.TB, gopts GlobalOptions) {
+	rtest.OK(t, withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+		return runRecover(context.TODO(), gopts, term)
+	}))
+}
+
+func TestRecover(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	// must list index more than once
+	env.gopts.backendTestHook = nil
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+
+	// create backup and forget it afterwards
+	testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts)
+	ids := testListSnapshots(t, env.gopts, 1)
+	sn := testLoadSnapshot(t, env.gopts, ids[0])
+	testRunForget(t, env.gopts, ForgetOptions{}, ids[0].String())
+	testListSnapshots(t, env.gopts, 0)
+
+	testRunRecover(t, env.gopts)
+	ids = testListSnapshots(t, env.gopts, 1)
+	testRunCheck(t, env.gopts)
+	// check that the root tree is included in the snapshot
+	rtest.OK(t, runCat(context.TODO(), env.gopts, []string{"tree", ids[0].String() + ":" + sn.Tree.Str()}))
+}
diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go
index 21944a9ce..dff84522a 100644
--- a/cmd/restic/integration_helpers_test.go
+++ b/cmd/restic/integration_helpers_test.go
@@ -354,6 +354,15 @@ func lastSnapshot(old, new map[string]struct{}) (map[string]struct{}, string) {
 	return old, ""
 }
 
+func testLoadSnapshot(t testing.TB, gopts GlobalOptions, id restic.ID) *restic.Snapshot {
+	_, repo, unlock, err := openWithReadLock(context.TODO(), gopts, false)
+	defer unlock()
+	rtest.OK(t, err)
+	snapshot, err := restic.LoadSnapshot(context.TODO(), repo, id)
+	rtest.OK(t, err)
+	return snapshot
+}
+
 func appendRandomData(filename string, bytes uint) error {
 	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0666)
 	if err != nil {
diff --git a/internal/backend/s3/s3_test.go b/internal/backend/s3/s3_test.go
index 40f7cb0b3..470088e07 100644
--- a/internal/backend/s3/s3_test.go
+++ b/internal/backend/s3/s3_test.go
@@ -118,7 +118,7 @@ func newMinioTestSuite(t testing.TB) (*test.Suite[s3.Config], func()) {
 			},
 
 			Factory: location.NewHTTPBackendFactory("s3", s3.ParseConfig, location.NoPassword, func(ctx context.Context, cfg s3.Config, rt http.RoundTripper) (be backend.Backend, err error) {
-				for i := 0; i < 50; i++ {
+				for i := 0; i < 10; i++ {
 					be, err = s3.Create(ctx, cfg, rt)
 					if err != nil {
 						t.Logf("s3 open: try %d: error %v", i, err)