Repository: kopia/kopia
Pull request #2379 (2022-09-07 04:53:01)

build(deps): bump cloud.google.com/go/storage from 1.25.0 to 1.26.0
Bumps [cloud.google.com/go/storage](https://github.com/googleapis/google-cloud-go) from 1.25.0 to 1.26.0.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a href="https://github.com/googleapis/google-cloud-go/releases">cloud.google.com/go/storage's releases</a>.</em></p>
<blockquote>
<h2>storage: v1.26.0</h2>
<h2><a href="https://github.com/googleapis/google-cloud-go/compare/storage/v1.25.0...storage/v1.26.0">1.26.0</a> (2022-08-29)</h2>
<h3>Features</h3>
<ul>
<li><strong>storage:</strong> export ShouldRetry (<a href="https://github-redirect.dependabot.com/googleapis/google-cloud-go/issues/6370">#6370</a>) (<a href="https://github.com/googleapis/google-cloud-go/commit/0da9ab0831540569dc04c0a23437b084b1564e15">0da9ab0</a>), refs <a href="https://github-redirect.dependabot.com/googleapis/google-cloud-go/issues/6362">#6362</a></li>
</ul>
<h3>Bug Fixes</h3>
<ul>
<li><strong>storage:</strong> allow to use age=0 in OLM conditions (<a href="https://github-redirect.dependabot.com/googleapis/google-cloud-go/issues/6204">#6204</a>) (<a href="https://github.com/googleapis/google-cloud-go/commit/c85704f4284626ce728cb48f3b130f2ce2a0165e">c85704f</a>)</li>
</ul>
<h2>pubsub: v1.25.1</h2>
<h2><a href="https://github.com/googleapis/google-cloud-go/compare/pubsub/v1.25.0...pubsub/v1.25.1">1.25.1</a> (2022-08-24)</h2>
<h3>Bug Fixes</h3>
<ul>
<li><strong>pubsub:</strong> up version of cloud.google.com/go (<a href="https://github-redirect.dependabot.com/googleapis/google-cloud-go/issues/6558">#6558</a>) (<a href="https://github.com/googleapis/google-cloud-go/commit/be9dcfbdfa5876a548eb3c60337c38e1d282bb88">be9dcfb</a>), refs <a href="https://github-redirect.dependabot.com/googleapis/google-cloud-go/issues/6555">#6555</a></li>
</ul>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a href="https://github.com/googleapis/google-cloud-go/commit/5edfd3693b2c4f76e04618d6453b486847a757da"><code>5edfd36</code></a> chore: release spanner 1.26.0 (<a href="https://github-redirect.dependabot.com/googleapis/google-cloud-go/issues/4751">#4751</a>)</li>
<li><a href="https://github.com/googleapis/google-cloud-go/commit/090cc3ae0f8747a14cc904fc6d429e2f5379bb03"><code>090cc3a</code></a> chore(all): auto-regenerate gapics (<a href="https://github-redirect.dependabot.com/googleapis/google-cloud-go/issues/4972">#4972</a>)</li>
<li><a href="https://github.com/googleapis/google-cloud-go/commit/acece394a38843f4355a82d0abab1e41994eca12"><code>acece39</code></a> chore(all): update all (<a href="https://github-redirect.dependabot.com/googleapis/google-cloud-go/issues/4971">#4971</a>)</li>
<li><a href="https://github.com/googleapis/google-cloud-go/commit/309b59e583d1bf0dd9ffe84223034eb8a2975d47"><code>309b59e</code></a> fix(internal/godocfx): only put TOC status on mod if all pkgs have same statu...</li>
<li><a href="https://github.com/googleapis/google-cloud-go/commit/3441e1f1b6b161c36cb14818eacd53af44c18a8a"><code>3441e1f</code></a> test(bigtable): abbrviate instance and cluster names to avoid length overrun ...</li>
<li><a href="https://github.com/googleapis/google-cloud-go/commit/26dff4fca81638e838a61d5c83991e41cefd1286"><code>26dff4f</code></a> chore: release storage 1.18.0 (<a href="https://github-redirect.dependabot.com/googleapis/google-cloud-go/issues/4951">#4951</a>)</li>
<li><a href="https://github.com/googleapis/google-cloud-go/commit/9e41088bb395fbae0e757738277d5c95fa2749c8"><code>9e41088</code></a> chore(all): auto-regenerate gapics (<a href="https://github-redirect.dependabot.com/googleapis/google-cloud-go/issues/4966">#4966</a>)</li>
<li><a href="https://github.com/googleapis/google-cloud-go/commit/5882cc7c6780f43d53fa67b84509bc5f4ea603c9"><code>5882cc7</code></a> test(bigtable): avoid instance name collisions to speed up creation (<a href="https://github-redirect.dependabot.com/googleapis/google-cloud-go/issues/4970">#4970</a>)</li>
<li><a href="https://github.com/googleapis/google-cloud-go/commit/5608b12141cee6abb0be1a1921696828297d005b"><code>5608b12</code></a> test(profiler): use debian 11 images for integration tests (<a href="https://github-redirect.dependabot.com/googleapis/google-cloud-go/issues/4965">#4965</a>)</li>
<li><a href="https://github.com/googleapis/google-cloud-go/commit/b824c897e6941270747b612f6d36a8d6ae081315"><code>b824c89</code></a> feat(storage): SignedUrl can use existing creds to authenticate (<a href="https://github-redirect.dependabot.com/googleapis/google-cloud-go/issues/4604">#4604</a>)</li>
<li>Additional commits viewable in <a href="https://github.com/googleapis/google-cloud-go/compare/pubsub/v1.25.0...spanner/v1.26.0">compare view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=cloud.google.com/go/storage&package-manager=go_modules&previous-version=1.25.0&new-version=1.26.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`.

[//]: # (dependabot-automerge-start)
Dependabot will merge this PR once it's up-to-date and CI passes on it, as requested by @kopia-builder.

[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually
- `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)


</details>

dependabot[bot]: One of your CI runs failed on this pull request, so Dependabot won't merge it.

Dependabot will still automatically merge this pull request if you amend it and your tests pass.
codecov[bot]: # [Codecov](https://codecov.io/gh/kopia/kopia/pull/2379?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kopia) Report
> Merging [#2379](https://codecov.io/gh/kopia/kopia/pull/2379?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kopia) (66ea43f) into [master](https://codecov.io/gh/kopia/kopia/commit/4353bfcec04577cc800770adf4038aa69fdfda93?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kopia) (4353bfc) will **increase** coverage by `0.02%`.
> The diff coverage is `n/a`.

```diff
@@            Coverage Diff             @@
##           master    #2379      +/-   ##
==========================================
+ Coverage   74.40%   74.42%   +0.02%     
==========================================
  Files         436      436              
  Lines       34522    34522              
==========================================
+ Hits        25685    25694       +9     
+ Misses       6963     6957       -6     
+ Partials     1874     1871       -3     
```


| [Impacted Files](https://codecov.io/gh/kopia/kopia/pull/2379?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kopia) | Coverage Δ | |
|---|---|---|
| [repo/blob/throttling/throttling\_semaphore.go](https://codecov.io/gh/kopia/kopia/pull/2379/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kopia#diff-cmVwby9ibG9iL3Rocm90dGxpbmcvdGhyb3R0bGluZ19zZW1hcGhvcmUuZ28=) | `97.05% <0.00%> (-2.95%)` | :arrow_down: |
| [snapshot/snapshotfs/repofs.go](https://codecov.io/gh/kopia/kopia/pull/2379/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kopia#diff-c25hcHNob3Qvc25hcHNob3Rmcy9yZXBvZnMuZ28=) | `79.29% <0.00%> (-1.52%)` | :arrow_down: |
| [repo/content/index/index\_v1.go](https://codecov.io/gh/kopia/kopia/pull/2379/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kopia#diff-cmVwby9jb250ZW50L2luZGV4L2luZGV4X3YxLmdv) | `75.00% <0.00%> (-1.18%)` | :arrow_down: |
| [snapshot/snapshotfs/upload.go](https://codecov.io/gh/kopia/kopia/pull/2379/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kopia#diff-c25hcHNob3Qvc25hcHNob3Rmcy91cGxvYWQuZ28=) | `83.40% <0.00%> (+0.66%)` | :arrow_up: |
| [internal/server/source\_manager.go](https://codecov.io/gh/kopia/kopia/pull/2379/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kopia#diff-aW50ZXJuYWwvc2VydmVyL3NvdXJjZV9tYW5hZ2VyLmdv) | `82.30% <0.00%> (+1.02%)` | :arrow_up: |
| [snapshot/policy/policy\_manager.go](https://codecov.io/gh/kopia/kopia/pull/2379/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kopia#diff-c25hcHNob3QvcG9saWN5L3BvbGljeV9tYW5hZ2VyLmdv) | `75.47% <0.00%> (+2.26%)` | :arrow_up: |

Help us with your feedback. Take ten seconds to tell us [how you rate us](https://about.codecov.io/nps?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kopia). Have a feature suggestion? [Share it here.](https://app.codecov.io/gh/feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kopia)


Patch:
diff --git a/.codecov.yml b/.codecov.yml
index ac94fc5e..fa4aeb64 100644
--- a/.codecov.yml
+++ b/.codecov.yml
@@ -25,4 +25,5 @@ ignore:
  - internal/repotesting/
  - internal/fshasher/
  - internal/blobtesting/
+ - internal/bigmap/bigmapbench/
  
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 3ff6b293..8fc40ca1 100644
--- a/Makefile
+++ b/Makefile
@@ -344,6 +344,10 @@ endif
 htmlui-e2e-test:
 	HTMLUI_E2E_TEST=1 go test -timeout 600s github.com/kopia/kopia/tests/htmlui_e2e_test -v $(TEST_FLAGS)
 
+htmlui-e2e-test-local-htmlui-changes:
+	(cd ../htmlui && npm run build)
+	HTMLUI_E2E_TEST=1 HTMLUI_BUILD_DIR=$(CURDIR)/../htmlui/build go test -timeout 600s github.com/kopia/kopia/tests/htmlui_e2e_test -v $(TEST_FLAGS)
+
 godoc:
 	godoc -http=:33333
 
diff --git a/app/package-lock.json b/app/package-lock.json
index 53b3cd93..cf5ce660 100644
--- a/app/package-lock.json
+++ b/app/package-lock.json
@@ -20,7 +20,7 @@
       "devDependencies": {
         "asar": "^3.2.0",
         "concurrently": "^7.3.0",
-        "dotenv": "^16.0.1",
+        "dotenv": "^16.0.2",
         "electron": "^20.0.1",
         "electron-builder": "^23.3.3",
         "electron-notarize": "^1.2.1"
@@ -1413,9 +1413,9 @@
       }
     },
     "node_modules/dotenv": {
-      "version": "16.0.1",
-      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.1.tgz",
-      "integrity": "sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ==",
+      "version": "16.0.2",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.2.tgz",
+      "integrity": "sha512-JvpYKUmzQhYoIFgK2MOnF3bciIZoItIIoryihy0rIA+H4Jy0FmgyKYAHCTN98P5ybGSJcIFbh6QKeJdtZd1qhA==",
       "dev": true,
       "engines": {
         "node": ">=12"
@@ -4708,9 +4708,9 @@
       }
     },
     "dotenv": {
-      "version": "16.0.1",
-      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.1.tgz",
-      "integrity": "sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ==",
+      "version": "16.0.2",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.2.tgz",
+      "integrity": "sha512-JvpYKUmzQhYoIFgK2MOnF3bciIZoItIIoryihy0rIA+H4Jy0FmgyKYAHCTN98P5ybGSJcIFbh6QKeJdtZd1qhA==",
       "dev": true
     },
     "dotenv-expand": {
diff --git a/app/package.json b/app/package.json
index 4bfb8d73..d524ac9f 100644
--- a/app/package.json
+++ b/app/package.json
@@ -112,7 +112,7 @@
   "devDependencies": {
     "asar": "^3.2.0",
     "concurrently": "^7.3.0",
-    "dotenv": "^16.0.1",
+    "dotenv": "^16.0.2",
     "electron": "^20.0.1",
     "electron-builder": "^23.3.3",
     "electron-notarize": "^1.2.1"
diff --git a/cli/command_snapshot_fix.go b/cli/command_snapshot_fix.go
index 51ad751f..336809b0 100644
--- a/cli/command_snapshot_fix.go
+++ b/cli/command_snapshot_fix.go
@@ -64,12 +64,16 @@ func failedEntryCallback(rep repo.RepositoryWriter, enumVal string) snapshotfs.R
 }
 
 func (c *commonRewriteSnapshots) rewriteMatchingSnapshots(ctx context.Context, rep repo.RepositoryWriter, rewrite snapshotfs.RewriteDirEntryCallback) error {
-	rw := snapshotfs.NewDirRewriter(rep, snapshotfs.DirRewriterOptions{
+	rw, err := snapshotfs.NewDirRewriter(ctx, rep, snapshotfs.DirRewriterOptions{
 		Parallel:               c.parallel,
 		RewriteEntry:           rewrite,
 		OnDirectoryReadFailure: failedEntryCallback(rep, c.invalidDirHandling),
 	})
-	defer rw.Close()
+	if err != nil {
+		return errors.Wrap(err, "unable to create dir rewriter")
+	}
+
+	defer rw.Close(ctx)
 
 	var updatedSnapshots int
 
diff --git a/go.mod b/go.mod
index 324c533a..64e0108a 100644
--- a/go.mod
+++ b/go.mod
@@ -3,13 +3,13 @@ module github.com/kopia/kopia
 go 1.17
 
 require (
-	cloud.google.com/go/storage v1.25.0
+	cloud.google.com/go/storage v1.26.0
 	github.com/Azure/azure-pipeline-go v0.2.3 // indirect
 	github.com/Azure/azure-storage-blob-go v0.15.0
 	github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 // indirect
 	github.com/alecthomas/kingpin v1.3.8-0.20220615105907-eae6867f4166 // this is pulling master, which is newer than v2
 	github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a
-	github.com/aws/aws-sdk-go v1.44.94
+	github.com/aws/aws-sdk-go v1.44.95
 	github.com/chmduquesne/rollinghash v4.0.0+incompatible
 	github.com/dustinkirkland/golang-petname v0.0.0-20191129215211-8e5a1ed0cff0
 	github.com/fatih/color v1.13.0
@@ -54,7 +54,7 @@ require (
 	golang.org/x/net v0.0.0-20220722155237-a158d28d115b
 	golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094
 	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
-	golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab
+	golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
 	golang.org/x/text v0.3.7
 	google.golang.org/api v0.95.0
@@ -83,7 +83,7 @@ require (
 	github.com/googleapis/gax-go/v2 v2.4.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/kopia/htmluibuild v0.0.0-20220817045620-d7803267445c
+	github.com/kopia/htmluibuild v0.0.0-20220909025317-0e48dcdc4db4
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/mattn/go-ieproxy v0.0.1 // indirect
 	github.com/mattn/go-isatty v0.0.16 // indirect
@@ -99,12 +99,12 @@ require (
 	go.uber.org/multierr v1.8.0
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220720214146-176da50484ac // indirect
+	google.golang.org/genproto v0.0.0-20220810155839-1856144b1d9c // indirect
 )
 
 require (
-	github.com/chromedp/cdproto v0.0.0-20220812200530-d0d83820bffc
-	github.com/chromedp/chromedp v0.8.4
+	github.com/chromedp/cdproto v0.0.0-20220827030233-358ed4af73cf
+	github.com/chromedp/chromedp v0.8.5
 	github.com/edsrzf/mmap-go v1.1.0
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/klauspost/reedsolomon v1.10.0
diff --git a/go.sum b/go.sum
index a60a98c2..9e62c144 100644
--- a/go.sum
+++ b/go.sum
@@ -57,8 +57,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y=
-cloud.google.com/go/storage v1.25.0 h1:D2Dn0PslpK7Z3B2AvuUHyIC762bDbGJdlmQlCBR71os=
-cloud.google.com/go/storage v1.25.0/go.mod h1:Qys4JU+jeup3QnuKKAosWuxrD95C4MSqxfVDnSirDsI=
+cloud.google.com/go/storage v1.26.0 h1:lYAGjknyDJirSzfwUlkv4Nsnj7od7foxQNH/fqZqles=
+cloud.google.com/go/storage v1.26.0/go.mod h1:mk/N7YwIKEWyTvXAWQCIeiCTdLoRH6Pd5xmSnolQLTI=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U=
 github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k=
@@ -101,8 +101,8 @@ github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8V
 github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=
 github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/aws/aws-sdk-go v1.44.94 h1:hDqJSv03ZVvqT448gUE63JEIHKx++vKLoDkiZxbNmIk=
-github.com/aws/aws-sdk-go v1.44.94/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/aws/aws-sdk-go v1.44.95 h1:QwmA+PeR6v4pF0f/dPHVPWGAshAhb9TnGZBTM5uKuI8=
+github.com/aws/aws-sdk-go v1.44.95/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -117,10 +117,10 @@ github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cb
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chmduquesne/rollinghash v4.0.0+incompatible h1:hnREQO+DXjqIw3rUTzWN7/+Dpw+N5Um8zpKV0JOEgbo=
 github.com/chmduquesne/rollinghash v4.0.0+incompatible/go.mod h1:Uc2I36RRfTAf7Dge82bi3RU0OQUmXT9iweIcPqvr8A0=
-github.com/chromedp/cdproto v0.0.0-20220812200530-d0d83820bffc h1:xGhCpiX5oNMoZGnzYvv1ne4muVRl2SDHH5fL7oUbZAY=
-github.com/chromedp/cdproto v0.0.0-20220812200530-d0d83820bffc/go.mod h1:5Y4sD/eXpwrChIuxhSr/G20n9CdbCmoerOHnuAf0Zr0=
-github.com/chromedp/chromedp v0.8.4 h1:50NSLCXC38kGsCmLCi8tXZ5V5FtQ3BOV/90u1O06Gq4=
-github.com/chromedp/chromedp v0.8.4/go.mod h1:ikuiJrEMoOnTPFUBu6jV0XZFOGW6W/Nhgk/OZyvh00o=
+github.com/chromedp/cdproto v0.0.0-20220827030233-358ed4af73cf h1:e0oJZmGJidTRZ0FvWXNhdj9OaOMN30Yf+T3K2Tf3L+s=
+github.com/chromedp/cdproto v0.0.0-20220827030233-358ed4af73cf/go.mod h1:5Y4sD/eXpwrChIuxhSr/G20n9CdbCmoerOHnuAf0Zr0=
+github.com/chromedp/chromedp v0.8.5 h1:HAVg54yQFcn7sg5reVjXtoI1eQaFxhjAjflHACicUFw=
+github.com/chromedp/chromedp v0.8.5/go.mod h1:xal2XY5Di7m/bzlGwtoYpmgIOfDqCakOIVg5OfdkPZ4=
 github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic=
 github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -341,8 +341,8 @@ github.com/klauspost/reedsolomon v1.10.0 h1:MonMtg979rxSHjwtsla5dZLhreS0Lu42AyQ2
 github.com/klauspost/reedsolomon v1.10.0/go.mod h1:qHMIzMkuZUWqIh8mS/GruPdo3u0qwX2jk/LH440ON7Y=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kopia/htmluibuild v0.0.0-20220817045620-d7803267445c h1:UCbhxRmJ2AoBaVtpv0xBjteJ7UtRJCNlbU9uu0/Aoyc=
-github.com/kopia/htmluibuild v0.0.0-20220817045620-d7803267445c/go.mod h1:eWer4rx9P8lJo2eKc+Q7AZ1dE1x1hJNdkbDFPzMu1Hw=
+github.com/kopia/htmluibuild v0.0.0-20220909025317-0e48dcdc4db4 h1:rfylAI7XtcqMXgcVANjHfSBZYdesDBptqP4z6Nup2wA=
+github.com/kopia/htmluibuild v0.0.0-20220909025317-0e48dcdc4db4/go.mod h1:eWer4rx9P8lJo2eKc+Q7AZ1dE1x1hJNdkbDFPzMu1Hw=
 github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -630,7 +630,6 @@ golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j
 golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
-golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
 golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 h1:2o1E+E8TpNLklK9nHiPiK1uzIYrIHt+cQx3ynCwq9V8=
 golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -722,8 +721,9 @@ golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64 h1:UiNENfZ8gDvpiWw7IpOMQ27spWmThO1RwwdQVbJahJM=
+golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -838,7 +838,7 @@ google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69
 google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
 google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg=
 google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o=
-google.golang.org/api v0.88.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
+google.golang.org/api v0.94.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI=
 google.golang.org/api v0.95.0 h1:d1c24AAS01DYqXreBeuVV7ewY/U8Mnhh47pwtsgVtYg=
 google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
@@ -929,8 +929,8 @@ google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljW
 google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
 google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
 google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
-google.golang.org/genproto v0.0.0-20220720214146-176da50484ac h1:EOa+Yrhx1C0O+4pHeXeWrCwdI0tWI6IfUU56Vebs9wQ=
-google.golang.org/genproto v0.0.0-20220720214146-176da50484ac/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE=
+google.golang.org/genproto v0.0.0-20220810155839-1856144b1d9c h1:IooGDWedfLC6KLczH/uduUsKQP42ZZYhKx+zd50L1Sk=
+google.golang.org/genproto v0.0.0-20220810155839-1856144b1d9c/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
diff --git a/internal/bigmap/bigmap_internal.go b/internal/bigmap/bigmap_internal.go
new file mode 100644
index 00000000..691acaf8
--- /dev/null
+++ b/internal/bigmap/bigmap_internal.go
@@ -0,0 +1,476 @@
+// Package bigmap implements a custom hashmap data structure where keys and values are binary
+// and keys are meant to be well-distributed hashes, such as content IDs, object IDs, etc.
+//
+// Unlike regular maps this map is limited to adding and getting elements but does not support
+// any iteration or deletion, but is much more efficient in terms of memory usage.
+//
+// Data for the hash table is stored in large contiguous memory blocks (for small sets)
+// and automatically spills over to memory-mapped files for larger sets using only 8 bytes
+// per key in RAM.
+package bigmap
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/edsrzf/mmap-go"
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+
+	"github.com/kopia/kopia/repo/logging"
+)
+
+const (
+	defaultNumMemorySegments    = 8          // number of segments to keep in RAM
+	defaultMemorySegmentSize    = 18 * 1e6   // 18MB enough to store >1M 16-17-byte keys
+	defaultFileSegmentSize      = 1024 << 20 // 1 GiB
+	defaultInitialSizeLogarithm = 20
+	mmapFileMode                = 0o600
+
+	// grow hash table above this percentage utilization, higher values (close to 100) will be very slow,
+	// smaller values will waste memory.
+	defaultLoadFactorPercentage = 75
+
+	minKeyLength = 4 // first 4 bytes of each key are used as uint32 hash value
+	maxKeyLength = 255
+
+	minSizeLogarithm = 8
+)
+
+var log = logging.Module("bigmap")
+
+// Options provides options for the internalMap.
+type Options struct {
+	LoadFactorPercentage int   // grow the size of the hash table when this percentage full
+	NumMemorySegments    int   // number of segments to keep in RAM
+	MemorySegmentSize    int64 // size of a single memory segment
+	FileSegmentSize      int   // size of a single file segment, defaults to 1 GiB
+
+	InitialSizeLogarithm int // logarithm of the initial size of the hash table, default - 20
+}
+
+// internalMap is a custom hashtable implementation using https://en.wikipedia.org/wiki/Double_hashing
+// that stores all (key,value) pairs densely packed in segments of fixed size to minimize data
+// fragmentation.
+type internalMap struct {
+	hasValues bool    // +checklocksignore
+	opts      Options // +checklocksignore
+
+	mu sync.RWMutex
+	// +checklocks:mu
+	tableSizeIndex int // index into tableSizesPrimes == len(slots)
+	// +checklocks:mu
+	segments []mmap.MMap // ([keyLength][key][valueLength][value])*
+	// +checklocks:mu
+	slots []entry // current hash table slots
+
+	// +checklocks:mu
+	count int // number of elements in the hash tables
+	// +checklocks:mu
+	h2Prime uint64 // prime < len(slots)
+
+	// +checklocks:mu
+	cleanups []func()
+
+	// +checklocks:mu
+	tempDir string
+}
+
+// The list of prime numbers close to 2^N from https://primes.utm.edu/lists/2small/0bit.html
+
+//nolint:gochecknoglobals
+var tableSizesPrimes = []uint64{
+	1<<8 - 59, // minSizeLogarithm
+	1<<9 - 55,
+	1<<10 - 57,
+	1<<11 - 61,
+	1<<12 - 77,
+	1<<13 - 91,
+	1<<14 - 111,
+	1<<15 - 135,
+	1<<16 - 123,
+	1<<17 - 99,
+	1<<18 - 93,
+	1<<19 - 87,
+	1<<20 - 185, // initial H2 prime
+	1<<21 - 129, // initial number of hashtable slots
+	1<<22 - 123,
+	1<<23 - 159,
+	1<<24 - 167,
+	1<<25 - 183,
+	1<<26 - 135,
+	1<<27 - 235,
+	1<<28 - 273,
+	1<<29 - 133,
+	1<<30 - 173,
+	1<<31 - 171,
+	1<<32 - 267,
+	1<<33 - 355,
+	1<<34 - 281,
+	1<<35 - 325,
+	1<<36 - 233,
+	1<<37 - 375,
+	1<<38 - 257,
+	1<<39 - 301,
+	1<<40 - 437,
+	1<<41 - 139,
+	1<<42 - 227,
+	1<<43 - 369,
+	1<<44 - 377,
+	1<<45 - 229,
+	1<<46 - 311,
+	1<<47 - 649,
+	1<<48 - 257,
+	1<<49 - 339,
+	1<<50 - 233,
+	1<<51 - 439,
+	1<<52 - 395,
+	1<<53 - 421,
+	1<<54 - 327,
+	1<<55 - 267,
+	1<<56 - 195,
+	1<<57 - 423,
+	1<<58 - 251,
+	1<<59 - 871,
+	1<<60 - 453,
+	1<<61 - 465,
+	1<<62 - 273,
+	1<<63 - 471,
+}
+
+type entry struct {
+	segment uint32 // 0-empty, otherwise index into internalMap.segments+1
+	offset  uint32 // offset within a segment
+}
+
+// Contains returns true if the provided key is in the map.
+func (m *internalMap) Contains(key []byte) bool {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	slot := m.findSlot(key)
+
+	return m.slots[slot].segment != 0
+}
+
+// Get gets the value associated with a given key. It is appended to the provided buffer.
+func (m *internalMap) Get(buf, key []byte) ([]byte, bool) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	slot := m.findSlot(key)
+	e := m.slots[slot]
+
+	if e.segment == 0 {
+		return nil, false
+	}
+
+	if !m.hasValues {
+		return nil, true
+	}
+
+	data := m.segments[e.segment-1] // 1-indexed
+	koff := e.offset
+
+	off := koff + uint32(data[koff]) + 1
+	vlen, vlenLen := binary.Uvarint(data[off:])
+	start := off + uint32(vlenLen)
+
+	return append(buf, data[start:start+uint32(vlen)]...), true
+}
+
+func (m *internalMap) hashValue(key []byte) uint64 {
+	if len(key) < 8 { //nolint:gomnd
+		return uint64(binary.BigEndian.Uint32(key))
+	}
+
+	return binary.BigEndian.Uint64(key)
+}
+
+// h2 returns the secondary hash value used for double hashing.
+func (m *internalMap) h2(key []byte) uint64 {
+	if len(key) < 16 { //nolint:gomnd
+		// use linear scan.
+		return 1
+	}
+
+	return binary.BigEndian.Uint64(key[8:])
+}
+
+// +checklocksread:m.mu
+func (m *internalMap) keyEquals(e entry, key []byte) bool {
+	data := m.segments[e.segment-1] // 1-indexed
+	koff := e.offset
+	keyLen := uint32(data[koff])
+
+	return bytes.Equal(key, data[koff+1:koff+1+keyLen])
+}
+
+// +checklocksread:m.mu
+func (m *internalMap) findSlotInSlice(key []byte, slots []entry, h2Prime uint64) uint64 {
+	slot := m.hashValue(key) % uint64(len(slots))
+
+	delta := m.h2(key) % h2Prime
+	if delta == 0 {
+		delta = 1
+	}
+
+	for slots[slot].segment != 0 && !m.keyEquals(slots[slot], key) {
+		slot = (slot + delta) % uint64(len(slots))
+	}
+
+	return slot
+}
+
+// +checklocksread:m.mu
+func (m *internalMap) findSlot(key []byte) uint64 {
+	return m.findSlotInSlice(key, m.slots, m.h2Prime)
+}
+
+// +checklocks:m.mu
+func (m *internalMap) growLocked(newSize uint64) {
+	newSlots := make([]entry, newSize)
+	newH2Prime := uint64(len(m.slots))
+
+	for segNum, seg := range m.segments {
+		p := 0
+
+		for p < len(seg) && seg[p] != 0 {
+			koff := p
+			key := seg[p+1 : p+1+int(seg[p])]
+			p += len(key) + 1
+
+			if m.hasValues {
+				vlen, vlenLen := binary.Uvarint(seg[p:])
+				p += vlenLen + int(vlen)
+			}
+
+			slot := m.findSlotInSlice(key, newSlots, newH2Prime)
+			newSlots[slot] = entry{segment: uint32(segNum) + 1, offset: uint32(koff)}
+		}
+	}
+
+	m.h2Prime = newH2Prime
+	m.slots = newSlots
+}
+
+// PutIfAbsent conditionally adds the provided (key, value) to the map if the provided key is absent.
+func (m *internalMap) PutIfAbsent(ctx context.Context, key, value []byte) bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if len(key) < minKeyLength {
+		panic("key too short")
+	}
+
+	if len(key) > maxKeyLength {
+		panic("key too long")
+	}
+
+	if !m.hasValues && len(value) > 0 {
+		panic("values are not enabled")
+	}
+
+	slot := m.findSlot(key)
+	if m.slots[slot].segment != 0 {
+		return false
+	}
+
+	if m.count >= len(m.slots)*m.opts.LoadFactorPercentage/100 {
+		m.tableSizeIndex++
+		m.growLocked(tableSizesPrimes[m.tableSizeIndex])
+		slot = m.findSlot(key)
+	}
+
+	payloadLength := 1 + len(key) + binary.MaxVarintLen32 + len(value)
+
+	current := m.segments[len(m.segments)-1]
+
+	// ensure that the key we're adding fits in the segment, start a new segment if not enough room.
+	if len(current)+payloadLength > cap(current) {
+		current = m.newSegment(ctx)
+		m.segments = append(m.segments, current)
+	}
+
+	koff := uint32(len(current))
+
+	current = append(current, byte(len(key)))
+	current = append(current, key...)
+
+	// append the value
+	if m.hasValues {
+		var vlen [binary.MaxVarintLen32]byte
+
+		n := binary.PutUvarint(vlen[:], uint64(len(value)))
+
+		current = append(current, vlen[0:n]...)
+		current = append(current, value...)
+	}
+
+	m.segments[len(m.segments)-1] = current
+	m.count++
+
+	m.slots[slot] = entry{
+		segment: uint32(len(m.segments)), // this is 1-based, 0==empty slot
+		offset:  koff,
+	}
+
+	return true
+}
+
+// +checklocks:m.mu
+func (m *internalMap) newMemoryMappedSegment(ctx context.Context) (mmap.MMap, error) {
+	flags := 0
+
+	f, err := m.maybeCreateMappedFile(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	s, err := mmap.MapRegion(f, m.opts.FileSegmentSize, mmap.RDWR, flags, 0)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to map region")
+	}
+
+	m.cleanups = append(m.cleanups, func() {
+		if err := s.Unmap(); err != nil {
+			log(ctx).Warnf("unable to unmap memory region: %v", err)
+		}
+	})
+
+	return s[:0], nil
+}
+
+// +checklocks:m.mu
+func (m *internalMap) maybeCreateMappedFile(ctx context.Context) (*os.File, error) {
+	if m.tempDir == "" {
+		tempDir, err := os.MkdirTemp("", "kopia-map")
+		if err != nil {
+			return nil, errors.Wrap(err, "unable to create temp directory")
+		}
+
+		m.tempDir = tempDir
+	}
+
+	fname := filepath.Join(m.tempDir, uuid.NewString())
+
+	f, err := os.OpenFile(fname, os.O_CREATE|os.O_EXCL|os.O_RDWR, mmapFileMode) //nolint:gosec
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to create memory-mapped file")
+	}
+
+	if err := f.Truncate(int64(m.opts.FileSegmentSize)); err != nil {
+		closeAndRemoveFile(ctx, f, fname)
+
+		return nil, errors.Wrap(err, "unable to truncate memory-mapped file")
+	}
+
+	m.cleanups = append(m.cleanups, func() {
+		closeAndRemoveFile(ctx, f, fname)
+	})
+
+	return f, nil
+}
+
+func closeAndRemoveFile(ctx context.Context, f *os.File, fname string) {
+	if err := f.Close(); err != nil {
+		log(ctx).Warnf("unable to close segment file: %v", err)
+	}
+
+	if err := os.Remove(fname); err != nil {
+		log(ctx).Warnf("unable to remove segment file: %v", err)
+	}
+}
+
+// +checklocks:m.mu
+func (m *internalMap) newSegment(ctx context.Context) mmap.MMap {
+	var s mmap.MMap
+
+	if len(m.segments) >= m.opts.NumMemorySegments {
+		var err error
+
+		s, err = m.newMemoryMappedSegment(ctx)
+		if err != nil {
+			log(ctx).Warnf("unable to create memory-mapped segment: %v", err)
+		}
+	}
+
+	if s == nil {
+		s = make([]byte, 0, m.opts.MemorySegmentSize)
+	}
+
+	return s
+}
+
+// Close releases all resources associated with a map.
+func (m *internalMap) Close(ctx context.Context) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	for i := len(m.cleanups) - 1; i >= 0; i-- {
+		m.cleanups[i]()
+	}
+
+	m.cleanups = nil
+	m.segments = nil
+
+	if m.tempDir != "" {
+		os.RemoveAll(m.tempDir) //nolint:errcheck
+	}
+}
+
+// newInternalMap creates new internalMap.
+func newInternalMap(ctx context.Context) (*internalMap, error) {
+	return newInternalMapWithOptions(ctx, true, nil)
+}
+
+// newInternalMapWithOptions creates a new instance of internalMap.
+func newInternalMapWithOptions(ctx context.Context, hasValues bool, opts *Options) (*internalMap, error) {
+	if opts == nil {
+		opts = &Options{}
+	}
+
+	if opts.LoadFactorPercentage == 0 {
+		opts.LoadFactorPercentage = defaultLoadFactorPercentage
+	}
+
+	if opts.MemorySegmentSize == 0 {
+		opts.MemorySegmentSize = defaultMemorySegmentSize
+	}
+
+	if opts.NumMemorySegments == 0 {
+		opts.NumMemorySegments = defaultNumMemorySegments
+	}
+
+	if opts.FileSegmentSize == 0 {
+		opts.FileSegmentSize = defaultFileSegmentSize
+	}
+
+	if opts.InitialSizeLogarithm == 0 {
+		opts.InitialSizeLogarithm = defaultInitialSizeLogarithm
+	}
+
+	tablewSizeIndex := opts.InitialSizeLogarithm - minSizeLogarithm
+
+	if tablewSizeIndex < 1 {
+		return nil, errors.Errorf("invalid initial size")
+	}
+
+	m := &internalMap{
+		hasValues:      hasValues,
+		opts:           *opts,
+		tableSizeIndex: tablewSizeIndex,
+		h2Prime:        tableSizesPrimes[tablewSizeIndex-1], // h2 prime < number of slots
+		slots:          make([]entry, tableSizesPrimes[tablewSizeIndex]),
+	}
+
+	m.mu.Lock()
+	m.segments = append(m.segments, m.newSegment(ctx))
+	m.mu.Unlock()
+
+	return m, nil
+}
diff --git a/internal/bigmap/bigmap_map.go b/internal/bigmap/bigmap_map.go
new file mode 100644
index 00000000..e210c677
--- /dev/null
+++ b/internal/bigmap/bigmap_map.go
@@ -0,0 +1,126 @@
+package bigmap
+
+import (
+	"context"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/binary"
+	"sync/atomic"
+
+	"github.com/pkg/errors"
+
+	"github.com/kopia/kopia/internal/gather"
+)
+
+const (
+	aesNonceSize = 12
+	aesKeySize   = 32 // AES-256
+)
+
+// Map is a wrapper around internalMap that adds encryption.
+type Map struct {
+	aead      cipher.AEAD
+	inner     *internalMap
+	nextNonce *uint64
+}
+
+// PutIfAbsent adds the element to the map, returns true if the element was added (as opposed to having
+// existed before).
+func (s *Map) PutIfAbsent(ctx context.Context, key, value []byte) bool {
+	if len(value) == 0 {
+		return s.inner.PutIfAbsent(ctx, key, nil)
+	}
+
+	var tmp gather.WriteBuffer
+	defer tmp.Close()
+
+	buf := tmp.MakeContiguous(aesNonceSize + len(value) + s.aead.Overhead())
+	nonce, input := buf[0:aesNonceSize], buf[aesNonceSize:aesNonceSize+len(value)]
+	copy(input, value)
+
+	nonceVal := atomic.AddUint64(s.nextNonce, 1)
+	if nonceVal == 0 {
+		panic("nonce counter wrapped beyond 64 bits, that should never happen")
+	}
+
+	binary.BigEndian.PutUint64(nonce, nonceVal)
+	s.aead.Seal(input[:0], nonce, input, key)
+
+	return s.inner.PutIfAbsent(ctx, key, buf)
+}
+
+// Get gets the element from the map and appends the value to the provided buffer.
+func (s *Map) Get(ctx context.Context, output, key []byte) (result []byte, ok bool, err error) {
+	if v, ok := s.inner.Get(output, key); ok {
+		result, err := s.decrypt(key, v)
+
+		return result, true, err
+	}
+
+	return nil, false, nil
+}
+
+func (s *Map) decrypt(key, buf []byte) ([]byte, error) {
+	if len(buf) == 0 {
+		return nil, nil
+	}
+
+	nonce, input := buf[0:aesNonceSize], buf[aesNonceSize:]
+
+	result, err := s.aead.Open(input[:0], nonce, input, key)
+	if err != nil {
+		return nil, errors.Errorf("unable to decrypt content: %v", err)
+	}
+
+	return result, nil
+}
+
+// Contains returns true if a given key exists in the set.
+func (s *Map) Contains(key []byte) bool {
+	return s.inner.Contains(key)
+}
+
+// Close releases resources associated with the set.
+func (s *Map) Close(ctx context.Context) {
+	s.inner.Close(ctx)
+}
+
+// NewMap creates new Map.
+func NewMap(ctx context.Context) (*Map, error) {
+	return NewMapWithOptions(ctx, nil)
+}
+
+// NewMapWithOptions creates new Map with options.
+func NewMapWithOptions(ctx context.Context, opt *Options) (*Map, error) {
+	inner, err := newInternalMapWithOptions(ctx, true, opt)
+	if err != nil {
+		return nil, err
+	}
+
+	key := make([]byte, aesKeySize)
+
+	if _, err = rand.Read(key); err != nil {
+		return nil, errors.Wrap(err, "error initializing map key")
+	}
+
+	enc, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, errors.Wrap(err, "error initializing map cipher")
+	}
+
+	aead, err := cipher.NewGCM(enc)
+	if err != nil {
+		return nil, errors.Wrap(err, "error initializing map AEAD")
+	}
+
+	if aead.NonceSize() != aesNonceSize {
+		return nil, errors.Errorf("unexpected nonce size: %v, expected %v", aead.NonceSize(), aesNonceSize)
+	}
+
+	return &Map{
+		aead:      aead,
+		inner:     inner,
+		nextNonce: new(uint64),
+	}, nil
+}
diff --git a/internal/bigmap/bigmap_set.go b/internal/bigmap/bigmap_set.go
new file mode 100644
index 00000000..a170245a
--- /dev/null
+++ b/internal/bigmap/bigmap_set.go
@@ -0,0 +1,39 @@
+package bigmap
+
+import "context"
+
+// Set is a wrapper around Map that only supports Put() and Contains().
+type Set struct {
+	inner *internalMap
+}
+
+// Put adds the element to a set, returns true if the element was added (as opposed to having
+// existed before).
+func (s *Set) Put(ctx context.Context, key []byte) bool {
+	return s.inner.PutIfAbsent(ctx, key, nil)
+}
+
+// Contains returns true if a given key exists in the set.
+func (s *Set) Contains(key []byte) bool {
+	return s.inner.Contains(key)
+}
+
+// Close releases resources associated with the set.
+func (s *Set) Close(ctx context.Context) {
+	s.inner.Close(ctx)
+}
+
+// NewSet creates new Set.
+func NewSet(ctx context.Context) (*Set, error) {
+	return NewSetWithOptions(ctx, nil)
+}
+
+// NewSetWithOptions creates new Set with options.
+func NewSetWithOptions(ctx context.Context, opt *Options) (*Set, error) {
+	inner, err := newInternalMapWithOptions(ctx, false, opt)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Set{inner}, nil
+}
diff --git a/internal/bigmap/bigmapbench/main.go b/internal/bigmap/bigmapbench/main.go
new file mode 100644
index 00000000..c276ee4e
--- /dev/null
+++ b/internal/bigmap/bigmapbench/main.go
@@ -0,0 +1,113 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/binary"
+	"flag"
+	"fmt"
+	"os"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/pkg/profile"
+
+	"github.com/kopia/kopia/internal/bigmap"
+	"github.com/kopia/kopia/internal/clock"
+	"github.com/kopia/kopia/repo/logging"
+)
+
+const (
+	implSyncMap           = 0
+	implMapWithEmptyValue = 1
+	implMapWithValues     = 2
+)
+
+//nolint:gochecknoglobals
+var (
+	impl          = flag.Int("impl", implMapWithEmptyValue, "Select implementation")
+	profileDir    = flag.String("profile-dir", "", "Profile directory")
+	profileCPU    = flag.Bool("profile-cpu", false, "Profile CPU")
+	profileMemory = flag.Bool("profile-memory", false, "Profile RAM")
+)
+
+func main() {
+	flag.Parse()
+
+	ctx := logging.WithLogger(context.Background(), logging.ToWriter(os.Stderr))
+
+	var (
+		bm     *bigmap.Map
+		sm     *sync.Map
+		num    [8]byte
+		keyBuf [sha256.Size]byte
+		ms0    runtime.MemStats
+	)
+
+	if *profileDir != "" {
+		pp := profile.ProfilePath(*profileDir)
+
+		if *profileCPU {
+			defer profile.Start(pp, profile.CPUProfile).Stop()
+		}
+
+		if *profileMemory {
+			defer profile.Start(pp, profile.MemProfile).Stop()
+		}
+	}
+
+	switch *impl {
+	case implSyncMap:
+		sm = &sync.Map{}
+
+		fmt.Println("using sync.Map")
+	case implMapWithEmptyValue:
+		fmt.Println("using bigmap.Map without values")
+
+		bm, _ = bigmap.NewMapWithOptions(ctx, &bigmap.Options{})
+	case implMapWithValues:
+		fmt.Println("using bigmap.Map with values")
+
+		bm, _ = bigmap.NewMapWithOptions(ctx, &bigmap.Options{})
+	}
+
+	h := sha256.New()
+
+	runtime.ReadMemStats(&ms0)
+
+	t0 := clock.Now()
+
+	for i := 0; i < 300_000_000; i++ {
+		if i%1_000_000 == 0 && i > 0 {
+			var ms runtime.MemStats
+
+			runtime.ReadMemStats(&ms)
+
+			alloc := ms.HeapAlloc - ms0.HeapAlloc
+			dur := clock.Now().Sub(t0).Truncate(time.Second)
+
+			fmt.Printf("elapsed %v count: %v M bytes: %v MB bytes/item: %v Mitems/sec: %.1f\n",
+				dur,
+				float64(i)/1e6,
+				alloc/1e6,
+				alloc/uint64(i),
+				float64(i)/dur.Seconds()/1e6)
+		}
+
+		// generate key=sha256(i) without allocations.
+		h.Reset()
+		binary.LittleEndian.PutUint64(num[:], uint64(i))
+		h.Write(num[:])
+		h.Sum(keyBuf[:0])
+
+		switch *impl {
+		case implSyncMap:
+			sm.LoadOrStore(keyBuf, nil)
+		case implMapWithEmptyValue:
+			bm.PutIfAbsent(ctx, keyBuf[:], nil)
+		case implMapWithValues:
+			bm.PutIfAbsent(ctx, keyBuf[:], keyBuf[:])
+		}
+	}
+}
diff --git a/repo/content/content_manager_iterate.go b/repo/content/content_manager_iterate.go
index fee5fbd7..af563f82 100644
--- a/repo/content/content_manager_iterate.go
+++ b/repo/content/content_manager_iterate.go
@@ -9,6 +9,7 @@
 
 	"github.com/pkg/errors"
 
+	"github.com/kopia/kopia/internal/bigmap"
 	"github.com/kopia/kopia/repo/blob"
 	"github.com/kopia/kopia/repo/content/index"
 )
@@ -228,7 +229,12 @@ func(ci Info) error {
 
 // IterateUnreferencedBlobs returns the list of unreferenced storage blobs.
 func (bm *WriteManager) IterateUnreferencedBlobs(ctx context.Context, blobPrefixes []blob.ID, parallellism int, callback func(blob.Metadata) error) error {
-	var usedPacks sync.Map
+	usedPacks, err := bigmap.NewSet(ctx)
+	if err != nil {
+		return errors.Wrap(err, "new set")
+	}
+
+	defer usedPacks.Close(ctx)
 
 	bm.log.Debugf("determining blobs in use")
 	// find packs in use
@@ -240,7 +246,7 @@ func (bm *WriteManager) IterateUnreferencedBlobs(ctx context.Context, blobPrefix
 		},
 		func(pi PackInfo) error {
 			if pi.ContentCount > 0 {
-				usedPacks.Store(pi.PackID, struct{}{})
+				usedPacks.Put(ctx, []byte(pi.PackID))
 			}
 			return nil
 		}); err != nil {
@@ -270,7 +276,7 @@ func(pi PackInfo) error {
 
 	if err := blob.IterateAllPrefixesInParallel(ctx, parallellism, bm.st, prefixes,
 		func(bm blob.Metadata) error {
-			if _, ok := usedPacks.Load(bm.BlobID); ok {
+			if usedPacks.Contains([]byte(bm.BlobID)) {
 				return nil
 			}
 
diff --git a/repo/content/index/id.go b/repo/content/index/id.go
index 430f49a0..e57e046e 100644
--- a/repo/content/index/id.go
+++ b/repo/content/index/id.go
@@ -108,6 +108,19 @@ func (i ID) AppendToLogBuffer(sb *logging.Buffer) {
 	sb.AppendBytes(buf[0 : i.idLen*2])
 }
 
+// Append appends content ID to the slice.
+func (i ID) Append(out []byte) []byte {
+	var buf [128]byte
+
+	if i.prefix != 0 {
+		out = append(out, i.prefix)
+	}
+
+	hex.Encode(buf[0:i.idLen*2], i.data[0:i.idLen])
+
+	return append(out, buf[0:i.idLen*2]...)
+}
+
 // String returns a string representation of ID.
 func (i ID) String() string {
 	return string(i.Prefix()) + hex.EncodeToString(i.data[:i.idLen])
diff --git a/repo/object/objectid.go b/repo/object/objectid.go
index 663ed750..d078cfd9 100644
--- a/repo/object/objectid.go
+++ b/repo/object/objectid.go
@@ -80,6 +80,19 @@ func (i ID) String() string {
 	return indirectPrefix + compressionPrefix + i.cid.String()
 }
 
+// Append appends string representation of ObjectID that is suitable for displaying in the UI.
+func (i ID) Append(out []byte) []byte {
+	for j := 0; j < int(i.indirection); j++ {
+		out = append(out, 'I')
+	}
+
+	if i.compression {
+		out = append(out, 'Z')
+	}
+
+	return i.cid.Append(out)
+}
+
 // IndexObjectID returns the object ID of the underlying index object.
 func (i ID) IndexObjectID() (ID, bool) {
 	if i.indirection > 0 {
diff --git a/snapshot/snapshotfs/dir_rewriter.go b/snapshot/snapshotfs/dir_rewriter.go
index a98176fc..5dbb6deb 100644
--- a/snapshot/snapshotfs/dir_rewriter.go
+++ b/snapshot/snapshotfs/dir_rewriter.go
@@ -7,10 +7,10 @@
 	"encoding/json"
 	"path"
 	"runtime"
-	"sync"
 
 	"github.com/pkg/errors"
 
+	"github.com/kopia/kopia/internal/bigmap"
 	"github.com/kopia/kopia/internal/impossible"
 	"github.com/kopia/kopia/internal/workshare"
 	"github.com/kopia/kopia/repo"
@@ -53,7 +53,7 @@ type DirRewriter struct {
 	ws   *workshare.Pool
 	opts DirRewriterOptions
 
-	cache sync.Map
+	cache *bigmap.Map
 
 	rep repo.RepositoryWriter
 }
@@ -91,9 +91,16 @@ func (rw *DirRewriter) getCachedReplacement(ctx context.Context, parentPath stri
 	key := rw.getCacheKey(input)
 
 	// see if we already processed this exact directory entry
-	if v, ok := rw.cache.Load(key); ok {
-		//nolint:forcetypeassert
-		return v.(*snapshot.DirEntry).Clone(), nil
+	cached, ok, err := rw.cache.Get(ctx, nil, key[:])
+	if err != nil {
+		return nil, errors.Wrap(err, "cache get")
+	}
+
+	if ok {
+		de := &snapshot.DirEntry{}
+		jerr := json.Unmarshal(cached, de)
+
+		return de, errors.Wrap(jerr, "json unmarshal")
 	}
 
 	// entry not cached yet, run rewriter
@@ -114,10 +121,14 @@ func (rw *DirRewriter) getCachedReplacement(ctx context.Context, parentPath stri
 		result = rep2
 	}
 
-	actual, _ := rw.cache.LoadOrStore(key, result.Clone())
+	v, err := json.Marshal(result)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to marshal JSON")
+	}
+
+	rw.cache.PutIfAbsent(ctx, key[:], v)
 
-	//nolint:forcetypeassert
-	return actual.(*snapshot.DirEntry), nil
+	return result, nil
 }
 
 func (rw *DirRewriter) processDirectory(ctx context.Context, pathFromRoot string, entry *snapshot.DirEntry) (*snapshot.DirEntry, error) {
@@ -232,8 +243,10 @@ func (rw *DirRewriter) RewriteSnapshotManifest(ctx context.Context, man *snapsho
 }
 
 // Close closes the rewriter.
-func (rw *DirRewriter) Close() {
+func (rw *DirRewriter) Close(ctx context.Context) {
 	rw.ws.Close()
+
+	rw.cache.Close(ctx)
 }
 
 // RewriteKeep is a callback that keeps the unreadable entry.
@@ -294,7 +307,7 @@ func RewriteRemove(ctx context.Context, parentPath string, entry *snapshot.DirEn
 }
 
 // NewDirRewriter creates a new directory rewriter.
-func NewDirRewriter(rep repo.RepositoryWriter, opts DirRewriterOptions) *DirRewriter {
+func NewDirRewriter(ctx context.Context, rep repo.RepositoryWriter, opts DirRewriterOptions) (*DirRewriter, error) {
 	if opts.Parallel == 0 {
 		opts.Parallel = runtime.NumCPU()
 	}
@@ -303,9 +316,15 @@ func NewDirRewriter(rep repo.RepositoryWriter, opts DirRewriterOptions) *DirRewr
 		opts.OnDirectoryReadFailure = RewriteFail
 	}
 
-	return &DirRewriter{
-		ws:   workshare.NewPool(opts.Parallel - 1),
-		opts: opts,
-		rep:  rep,
+	cache, err := bigmap.NewMap(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "new map")
 	}
+
+	return &DirRewriter{
+		ws:    workshare.NewPool(opts.Parallel - 1),
+		opts:  opts,
+		rep:   rep,
+		cache: cache,
+	}, nil
 }
diff --git a/snapshot/snapshotfs/snapshot_storage_stats.go b/snapshot/snapshotfs/snapshot_storage_stats.go
index b5f06c3a..d9ac7304 100644
--- a/snapshot/snapshotfs/snapshot_storage_stats.go
+++ b/snapshot/snapshotfs/snapshot_storage_stats.go
@@ -2,13 +2,13 @@
 
 import (
 	"context"
-	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/pkg/errors"
 
 	"github.com/kopia/kopia/fs"
+	"github.com/kopia/kopia/internal/bigmap"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/object"
 	"github.com/kopia/kopia/snapshot"
@@ -24,9 +24,14 @@ func CalculateStorageStats(ctx context.Context, rep repo.Repository, manifests [
 	unique := new(snapshot.StorageUsageDetails)
 	runningTotal := new(snapshot.StorageUsageDetails)
 
-	var uniqueContents sync.Map
+	uniqueContents, err := bigmap.NewSet(ctx)
+	if err != nil {
+		return errors.Wrap(err, "NewSet")
+	}
+
+	defer uniqueContents.Close(ctx)
 
-	tw := NewTreeWalker(TreeWalkerOptions{
+	tw, twerr := NewTreeWalker(ctx, TreeWalkerOptions{
 		EntryCallback: func(ctx context.Context, entry fs.Entry, oid object.ID, entryPath string) error {
 			if !entry.IsDir() {
 				atomic.AddInt32(&unique.FileObjectCount, 1)
@@ -44,8 +49,10 @@ func CalculateStorageStats(ctx context.Context, rep repo.Repository, manifests [
 				return errors.Wrapf(err, "error verifying object %v", oid)
 			}
 
+			var cidbuf [128]byte
+
 			for _, cid := range contentIDs {
-				if _, loaded := uniqueContents.LoadOrStore(cid, struct{}{}); !loaded {
+				if uniqueContents.Put(ctx, cid.Append(cidbuf[:0])) {
 					atomic.AddInt32(&unique.ContentCount, 1)
 					atomic.AddInt32(&runningTotal.ContentCount, 1)
 
@@ -71,7 +78,10 @@ func CalculateStorageStats(ctx context.Context, rep repo.Repository, manifests [
 			return nil
 		},
 	})
-	defer tw.Close()
+	if twerr != nil {
+		return errors.Wrap(twerr, "tree walker")
+	}
+	defer tw.Close(ctx)
 
 	src := manifests[0].Source
 
diff --git a/snapshot/snapshotfs/snapshot_tree_walker.go b/snapshot/snapshotfs/snapshot_tree_walker.go
index 9757bd03..94a80d4e 100644
--- a/snapshot/snapshotfs/snapshot_tree_walker.go
+++ b/snapshot/snapshotfs/snapshot_tree_walker.go
@@ -9,6 +9,7 @@
 	"github.com/pkg/errors"
 
 	"github.com/kopia/kopia/fs"
+	"github.com/kopia/kopia/internal/bigmap"
 	"github.com/kopia/kopia/internal/workshare"
 	"github.com/kopia/kopia/repo/object"
 )
@@ -23,7 +24,7 @@
 type TreeWalker struct {
 	options TreeWalkerOptions
 
-	enqueued sync.Map
+	enqueued *bigmap.Set
 	wp       *workshare.Pool
 
 	mu sync.Mutex
@@ -85,9 +86,10 @@ func (w *TreeWalker) TooManyErrors() bool {
 	return w.numErrors >= w.options.MaxErrors
 }
 
-func (w *TreeWalker) alreadyProcessed(e fs.Entry) bool {
-	_, existing := w.enqueued.LoadOrStore(oidOf(e), struct{}{})
-	return existing
+func (w *TreeWalker) alreadyProcessed(ctx context.Context, e fs.Entry) bool {
+	var idbuf [128]byte
+
+	return !w.enqueued.Put(ctx, oidOf(e).Append(idbuf[:0]))
 }
 
 func (w *TreeWalker) processEntry(ctx context.Context, e fs.Entry, entryPath string) {
@@ -117,7 +119,7 @@ type errStop struct {
 			return errStop{errors.New("")}
 		}
 
-		if w.alreadyProcessed(ent) {
+		if w.alreadyProcessed(ctx, ent) {
 			return nil
 		}
 
@@ -146,7 +148,7 @@ func (w *TreeWalker) Process(ctx context.Context, e fs.Entry, entryPath string)
 		return errors.Errorf("entry does not have ObjectID")
 	}
 
-	if w.alreadyProcessed(e) {
+	if w.alreadyProcessed(ctx, e) {
 		return nil
 	}
 
@@ -156,8 +158,9 @@ func (w *TreeWalker) Process(ctx context.Context, e fs.Entry, entryPath string)
 }
 
 // Close closes the tree walker.
-func (w *TreeWalker) Close() {
+func (w *TreeWalker) Close(ctx context.Context) {
 	w.wp.Close()
+	w.enqueued.Close(ctx)
 }
 
 // TreeWalkerOptions provides optional fields for TreeWalker.
@@ -169,7 +172,7 @@ type TreeWalkerOptions struct {
 }
 
 // NewTreeWalker creates new tree walker.
-func NewTreeWalker(options TreeWalkerOptions) *TreeWalker {
+func NewTreeWalker(ctx context.Context, options TreeWalkerOptions) (*TreeWalker, error) {
 	if options.Parallelism <= 0 {
 		options.Parallelism = runtime.NumCPU() * walkersPerCPU
 	}
@@ -178,8 +181,14 @@ func NewTreeWalker(options TreeWalkerOptions) *TreeWalker {
 		options.MaxErrors = 1
 	}
 
-	return &TreeWalker{
-		options: options,
-		wp:      workshare.NewPool(options.Parallelism - 1),
+	s, err := bigmap.NewSet(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "NewSet")
 	}
+
+	return &TreeWalker{
+		options:  options,
+		wp:       workshare.NewPool(options.Parallelism - 1),
+		enqueued: s,
+	}, nil
 }
diff --git a/snapshot/snapshotfs/snapshot_verifier.go b/snapshot/snapshotfs/snapshot_verifier.go
index 290745e1..5b376a11 100644
--- a/snapshot/snapshotfs/snapshot_verifier.go
+++ b/snapshot/snapshotfs/snapshot_verifier.go
@@ -129,12 +129,15 @@ type VerifierOptions struct {
 // InParallel starts parallel verification and invokes the provided function which can call
 // call Process() on in the provided TreeWalker.
 func (v *Verifier) InParallel(ctx context.Context, enqueue func(tw *TreeWalker) error) error {
-	tw := NewTreeWalker(TreeWalkerOptions{
+	tw, twerr := NewTreeWalker(ctx, TreeWalkerOptions{
 		Parallelism:   v.opts.Parallelism,
 		EntryCallback: v.verifyObject,
 		MaxErrors:     v.opts.MaxErrors,
 	})
-	defer tw.Close()
+	if twerr != nil {
+		return errors.Wrap(twerr, "tree walker")
+	}
+	defer tw.Close(ctx)
 
 	v.fileWorkQueue = make(chan verifyFileWorkItem, v.opts.FileQueueLength)
 
diff --git a/snapshot/snapshotgc/gc.go b/snapshot/snapshotgc/gc.go
index 318fed96..ef2c400e 100644
--- a/snapshot/snapshotgc/gc.go
+++ b/snapshot/snapshotgc/gc.go
@@ -3,12 +3,12 @@
 
 import (
 	"context"
-	"sync"
 	"time"
 
 	"github.com/pkg/errors"
 
 	"github.com/kopia/kopia/fs"
+	"github.com/kopia/kopia/internal/bigmap"
 	"github.com/kopia/kopia/internal/stats"
 	"github.com/kopia/kopia/internal/units"
 	"github.com/kopia/kopia/repo"
@@ -23,7 +23,7 @@
 
 var log = logging.Module("snapshotgc")
 
-func findInUseContentIDs(ctx context.Context, rep repo.Repository, used *sync.Map) error {
+func findInUseContentIDs(ctx context.Context, rep repo.Repository, used *bigmap.Set) error {
 	ids, err := snapshot.ListSnapshotManifests(ctx, rep, nil, nil)
 	if err != nil {
 		return errors.Wrap(err, "unable to list snapshot manifest IDs")
@@ -34,21 +34,27 @@ func findInUseContentIDs(ctx context.Context, rep repo.Repository, used *sync.Ma
 		return errors.Wrap(err, "unable to load manifest IDs")
 	}
 
-	w := snapshotfs.NewTreeWalker(snapshotfs.TreeWalkerOptions{
+	w, twerr := snapshotfs.NewTreeWalker(ctx, snapshotfs.TreeWalkerOptions{
 		EntryCallback: func(ctx context.Context, entry fs.Entry, oid object.ID, entryPath string) error {
-			contentIDs, err := rep.VerifyObject(ctx, oid)
-			if err != nil {
-				return errors.Wrapf(err, "error verifying %v", oid)
+			contentIDs, verr := rep.VerifyObject(ctx, oid)
+			if verr != nil {
+				return errors.Wrapf(verr, "error verifying %v", oid)
 			}
 
+			var cidbuf [128]byte
+
 			for _, cid := range contentIDs {
-				used.Store(cid, nil)
+				used.Put(ctx, cid.Append(cidbuf[:0]))
 			}
 
 			return nil
 		},
 	})
-	defer w.Close()
+	if twerr != nil {
+		return errors.Wrap(err, "unable to create tree walker")
+	}
+
+	defer w.Close(ctx)
 
 	log(ctx).Infof("Looking for active contents...")
 
@@ -93,13 +99,15 @@ func Run(ctx context.Context, rep repo.DirectRepositoryWriter, gcDelete bool, sa
 }
 
 func runInternal(ctx context.Context, rep repo.DirectRepositoryWriter, gcDelete bool, safety maintenance.SafetyParameters, maintenanceStartTime time.Time, st *Stats) error {
-	var (
-		used sync.Map
+	var unused, inUse, system, tooRecent, undeleted stats.CountSum
 
-		unused, inUse, system, tooRecent, undeleted stats.CountSum
-	)
+	used, serr := bigmap.NewSet(ctx)
+	if serr != nil {
+		return errors.Wrap(serr, "unable to create new set")
+	}
+	defer used.Close(ctx)
 
-	if err := findInUseContentIDs(ctx, rep, &used); err != nil {
+	if err := findInUseContentIDs(ctx, rep, used); err != nil {
 		return errors.Wrap(err, "unable to find in-use content ID")
 	}
 
@@ -113,7 +121,9 @@ func runInternal(ctx context.Context, rep repo.DirectRepositoryWriter, gcDelete
 			return nil
 		}
 
-		if _, ok := used.Load(ci.GetContentID()); ok {
+		var cidbuf [128]byte
+
+		if used.Contains(ci.GetContentID().Append(cidbuf[:0])) {
 			if ci.GetDeleted() {
 				if err := rep.ContentManager().UndeleteContent(ctx, ci.GetContentID()); err != nil {
 					return errors.Wrapf(err, "Could not undelete referenced content: %v", ci)


Test patch:
diff --git a/internal/bigmap/bigmap_internal_test.go b/internal/bigmap/bigmap_internal_test.go
new file mode 100644
index 00000000..6c3b868a
--- /dev/null
+++ b/internal/bigmap/bigmap_internal_test.go
@@ -0,0 +1,290 @@
+package bigmap
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/binary"
+	"hash"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/kopia/kopia/internal/testlogging"
+)
+
+func TestInternalMap(t *testing.T) {
+	ctx := testlogging.Context(t)
+
+	impl, err := newInternalMap(ctx)
+	require.NoError(t, err)
+
+	defer impl.Close(ctx)
+
+	key1 := []byte("key1")
+	key2 := []byte("longerkey2")
+	val1 := []byte("val1")
+	val2 := []byte("val2")
+
+	v, ok := impl.Get(nil, key1)
+	require.Nil(t, v)
+	require.False(t, ok)
+
+	impl.PutIfAbsent(ctx, key1, val1)
+
+	v, ok = impl.Get(nil, key1)
+	require.True(t, ok)
+	require.Equal(t, val1, v)
+
+	v, ok = impl.Get(nil, key2)
+	require.Nil(t, v)
+	require.False(t, ok)
+
+	impl.PutIfAbsent(ctx, key2, val2)
+
+	v, ok = impl.Get(nil, key2)
+	require.True(t, ok)
+	require.Equal(t, val2, v)
+
+	v, ok = impl.Get(nil, key1)
+	require.True(t, ok)
+	require.Equal(t, val1, v)
+}
+
+func TestGrowingMap(t *testing.T) {
+	ctx := testlogging.Context(t)
+
+	impl, err := newInternalMapWithOptions(ctx, true, &Options{
+		InitialSizeLogarithm: 9,
+		NumMemorySegments:    3,
+		MemorySegmentSize:    1000,
+		FileSegmentSize:      4 << 20,
+	})
+	require.NoError(t, err)
+
+	defer impl.Close(ctx)
+
+	h := sha256.New()
+
+	// insert 20K hashes
+	for i := 0; i < 20000; i++ {
+		var keybuf, valbuf, valbuf2 [sha256.Size]byte
+
+		k := sha256Key(h, keybuf[:0], i)
+		v := sha256Key(h, valbuf[:0], i+3)
+
+		require.True(t, impl.PutIfAbsent(ctx, k, v))
+
+		// ensure that previously written key is still there.
+		pkindex := i / 2
+		pk := sha256Key(h, keybuf[:0], pkindex)
+		require.True(t, impl.Contains(pk))
+
+		pv, ok := impl.Get(valbuf2[:0], pk)
+		require.True(t, ok)
+		require.Equal(t, pv, sha256Key(h, valbuf[:0], pkindex+3))
+
+		// ensure that key not written yet is not there.
+		nk := sha256Key(h, keybuf[:0], i+1)
+
+		require.False(t, impl.Contains(nk))
+
+		_, ok2 := impl.Get(valbuf2[:0], nk)
+		require.False(t, ok2)
+	}
+}
+
+func TestGrowingSet(t *testing.T) {
+	ctx := testlogging.Context(t)
+
+	impl, err := NewSetWithOptions(ctx, &Options{
+		InitialSizeLogarithm: 9,
+		NumMemorySegments:    3,
+		MemorySegmentSize:    1000,
+		FileSegmentSize:      4 << 20,
+	})
+	require.NoError(t, err)
+
+	defer impl.Close(ctx)
+
+	h := sha256.New()
+
+	// insert 20K hashes
+	for i := 0; i < 20000; i++ {
+		var keybuf [sha256.Size]byte
+
+		k := sha256Key(h, keybuf[:0], i)
+
+		require.True(t, impl.Put(ctx, k))
+		require.False(t, impl.Put(ctx, k))
+
+		// ensure that previously written key is still there.
+		pkindex := i / 2
+		pk := sha256Key(h, keybuf[:0], pkindex)
+		require.True(t, impl.Contains(pk))
+
+		// ensure that key not written yet is not there.
+		nk := sha256Key(h, keybuf[:0], i+1)
+
+		require.False(t, impl.Contains(nk))
+	}
+}
+
+func sha256Key(h hash.Hash, out []byte, i int) []byte {
+	var num [8]byte
+
+	// generate key=sha256(i) without allocations.
+	h.Reset()
+	binary.LittleEndian.PutUint64(num[:], uint64(i))
+	h.Write(num[:])
+
+	s := h.Sum(out)
+
+	return s
+}
+
+func BenchmarkInternalMap_NoValue(b *testing.B) {
+	ctx := testlogging.Context(b)
+	m, err := newInternalMapWithOptions(ctx, false, nil)
+	require.NoError(b, err)
+
+	defer m.Close(ctx)
+
+	benchmarkInternalMap(b, m, []byte{})
+}
+
+func BenchmarkInternalMap_WithValue(b *testing.B) {
+	ctx := testlogging.Context(b)
+	m, err := newInternalMap(ctx)
+	require.NoError(b, err)
+
+	defer m.Close(ctx)
+
+	benchmarkInternalMap(b, m, []byte{1, 2, 3})
+}
+
+//nolint:thelper
+func benchmarkInternalMap(b *testing.B, m *internalMap, someVal []byte) {
+	ctx := testlogging.Context(b)
+
+	b.ResetTimer()
+
+	var (
+		h      = sha256.New()
+		num    [8]byte
+		keyBuf [sha256.Size]byte
+	)
+
+	for i := 0; i < b.N; i++ {
+		// generate key=sha256(i) without allocations.
+		h.Reset()
+		binary.LittleEndian.PutUint64(num[:], uint64(i))
+		h.Write(num[:])
+		key := h.Sum(keyBuf[:0])
+
+		m.PutIfAbsent(ctx, key, someVal)
+	}
+
+	valBuf := make([]byte, 10)
+
+	for j := 0; j < 4; j++ {
+		for i := 0; i < b.N; i++ {
+			// generate key=sha256(i) without allocations.
+			h.Reset()
+			binary.LittleEndian.PutUint64(num[:], uint64(i))
+			h.Write(num[:])
+			key := h.Sum(keyBuf[:0])
+
+			_, ok := m.Get(valBuf[:0], key)
+			require.True(b, ok)
+		}
+	}
+}
+
+func BenchmarkSyncMap_NoValue(b *testing.B) {
+	benchmarkSyncMap(b, []byte{})
+}
+
+func BenchmarkSyncMap_WithValue(b *testing.B) {
+	someVal := []byte{1, 2, 3}
+
+	benchmarkSyncMap(b, someVal)
+}
+
+//nolint:thelper
+func benchmarkSyncMap(b *testing.B, someVal []byte) {
+	var m sync.Map
+
+	var (
+		h      = sha256.New()
+		num    [8]byte
+		keyBuf [sha256.Size]byte
+	)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		// generate key=sha256(i) without allocations.
+		h.Reset()
+		binary.LittleEndian.PutUint64(num[:], uint64(i))
+		h.Write(num[:])
+		key := h.Sum(keyBuf[:0])
+
+		m.Store(string(key), append([]byte{}, someVal...))
+	}
+
+	for j := 0; j < 4; j++ {
+		for i := 0; i < b.N; i++ {
+			// generate key=sha256(i) without allocations.
+			h.Reset()
+			binary.LittleEndian.PutUint64(num[:], uint64(i))
+			h.Write(num[:])
+			key := h.Sum(keyBuf[:0])
+
+			val, ok := m.Load(string(key))
+			require.True(b, ok)
+			require.Equal(b, someVal, val)
+		}
+	}
+}
+
+func TestErrors(t *testing.T) {
+	ctx := testlogging.Context(t)
+
+	_, err := newInternalMapWithOptions(ctx, true, &Options{
+		InitialSizeLogarithm: 8,
+	})
+
+	require.ErrorContains(t, err, "invalid initial size")
+}
+
+func TestPanics(t *testing.T) {
+	ctx := testlogging.Context(t)
+
+	m, err := NewSet(ctx)
+	require.NoError(t, err)
+
+	// too short keys
+	require.Panics(t, func() { m.Put(ctx, nil) })
+	require.Panics(t, func() { m.Put(ctx, []byte{1}) })
+
+	// too long key
+	require.Panics(t, func() { m.Put(ctx, bytes.Repeat([]byte{1}, 256)) })
+}
+
+func TestMapWithoutValue(t *testing.T) {
+	ctx := testlogging.Context(t)
+
+	// this is a corner case, it's possible to create a map that can only support zero-length values.
+	m, err := newInternalMapWithOptions(ctx, false, nil)
+	require.NoError(t, err)
+
+	key := []byte{1, 2, 3, 4}
+	m.PutIfAbsent(ctx, key, nil)
+
+	require.Panics(t, func() { m.PutIfAbsent(ctx, key, []byte{3, 4, 5}) })
+
+	v, ok := m.Get(nil, key)
+	require.True(t, ok)
+	require.Nil(t, v)
+}
diff --git a/internal/bigmap/bigmap_map_test.go b/internal/bigmap/bigmap_map_test.go
new file mode 100644
index 00000000..b72d7bd5
--- /dev/null
+++ b/internal/bigmap/bigmap_map_test.go
@@ -0,0 +1,130 @@
+package bigmap_test
+
+import (
+	"crypto/sha256"
+	"encoding/binary"
+	"hash"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/kopia/kopia/internal/bigmap"
+	"github.com/kopia/kopia/internal/testlogging"
+)
+
+func TestGrowingMap(t *testing.T) {
+	ctx := testlogging.Context(t)
+
+	impl, err := bigmap.NewMapWithOptions(ctx, &bigmap.Options{
+		InitialSizeLogarithm: 9,
+		NumMemorySegments:    3,
+		MemorySegmentSize:    1000,
+		FileSegmentSize:      4 << 20,
+	})
+	require.NoError(t, err)
+
+	defer impl.Close(ctx)
+
+	h := sha256.New()
+
+	// insert 20K hashes
+	for i := 0; i < 20000; i++ {
+		var keybuf, valbuf, valbuf2 [sha256.Size]byte
+
+		k := sha256Key(h, keybuf[:0], i)
+		v := sha256Key(h, valbuf[:0], i+3)
+
+		require.True(t, impl.PutIfAbsent(ctx, k, v))
+
+		// ensure that previously written key is still there.
+		pkindex := i / 2
+		pk := sha256Key(h, keybuf[:0], pkindex)
+		require.True(t, impl.Contains(pk))
+
+		pv, ok, err := impl.Get(ctx, valbuf2[:0], pk)
+		require.NoError(t, err)
+		require.True(t, ok)
+		require.Equal(t, pv, sha256Key(h, valbuf[:0], pkindex+3))
+
+		// ensure that key not written yet is not there.
+		nk := sha256Key(h, keybuf[:0], i+1)
+
+		require.False(t, impl.Contains(nk))
+
+		_, ok2, err := impl.Get(ctx, valbuf2[:0], nk)
+		require.NoError(t, err)
+		require.False(t, ok2)
+	}
+}
+
+func sha256Key(h hash.Hash, out []byte, i int) []byte {
+	var num [8]byte
+
+	// generate key=sha256(i) without allocations.
+	h.Reset()
+	binary.LittleEndian.PutUint64(num[:], uint64(i))
+	h.Write(num[:])
+
+	s := h.Sum(out)
+
+	return s
+}
+
+func BenchmarkMap_NoValue(b *testing.B) {
+	ctx := testlogging.Context(b)
+	m, err := bigmap.NewMapWithOptions(ctx, nil)
+	require.NoError(b, err)
+
+	defer m.Close(ctx)
+
+	benchmarkMap(b, m, []byte{})
+}
+
+func BenchmarkMap_WithValue(b *testing.B) {
+	ctx := testlogging.Context(b)
+	m, err := bigmap.NewMap(ctx)
+	require.NoError(b, err)
+
+	defer m.Close(ctx)
+
+	benchmarkMap(b, m, []byte{1, 2, 3})
+}
+
+//nolint:thelper
+func benchmarkMap(b *testing.B, m *bigmap.Map, someVal []byte) {
+	ctx := testlogging.Context(b)
+
+	b.ResetTimer()
+
+	var (
+		h      = sha256.New()
+		num    [8]byte
+		keyBuf [sha256.Size]byte
+	)
+
+	for i := 0; i < b.N; i++ {
+		// generate key=sha256(i) without allocations.
+		h.Reset()
+		binary.LittleEndian.PutUint64(num[:], uint64(i))
+		h.Write(num[:])
+		key := h.Sum(keyBuf[:0])
+
+		m.PutIfAbsent(ctx, key, someVal)
+	}
+
+	valBuf := make([]byte, 10)
+
+	for j := 0; j < 4; j++ {
+		for i := 0; i < b.N; i++ {
+			// generate key=sha256(i) without allocations.
+			h.Reset()
+			binary.LittleEndian.PutUint64(num[:], uint64(i))
+			h.Write(num[:])
+			key := h.Sum(keyBuf[:0])
+
+			_, ok, err := m.Get(ctx, valBuf[:0], key)
+			require.NoError(b, err)
+			require.True(b, ok)
+		}
+	}
+}
diff --git a/internal/bigmap/bigmap_set_test.go b/internal/bigmap/bigmap_set_test.go
new file mode 100644
index 00000000..12bada23
--- /dev/null
+++ b/internal/bigmap/bigmap_set_test.go
@@ -0,0 +1,101 @@
+package bigmap_test
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/binary"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/kopia/kopia/internal/bigmap"
+	"github.com/kopia/kopia/internal/testlogging"
+)
+
+func TestGrowingSet(t *testing.T) {
+	ctx := testlogging.Context(t)
+
+	impl, err := bigmap.NewSetWithOptions(ctx, &bigmap.Options{
+		InitialSizeLogarithm: 9,
+		NumMemorySegments:    3,
+		MemorySegmentSize:    1000,
+		FileSegmentSize:      4 << 20,
+	})
+	require.NoError(t, err)
+
+	defer impl.Close(ctx)
+
+	h := sha256.New()
+
+	// insert 20K hashes
+	for i := 0; i < 20000; i++ {
+		var keybuf [sha256.Size]byte
+
+		k := sha256Key(h, keybuf[:0], i)
+
+		require.True(t, impl.Put(ctx, k))
+		require.False(t, impl.Put(ctx, k))
+
+		// ensure that previously written key is still there.
+		pkindex := i / 2
+		pk := sha256Key(h, keybuf[:0], pkindex)
+		require.True(t, impl.Contains(pk))
+
+		// ensure that key not written yet is not there.
+		nk := sha256Key(h, keybuf[:0], i+1)
+
+		require.False(t, impl.Contains(nk))
+	}
+}
+
+func BenchmarkSet(b *testing.B) {
+	ctx := testlogging.Context(b)
+	m, err := bigmap.NewSet(ctx)
+	require.NoError(b, err)
+
+	defer m.Close(ctx)
+
+	b.ResetTimer()
+
+	var (
+		h      = sha256.New()
+		num    [8]byte
+		keyBuf [sha256.Size]byte
+	)
+
+	for i := 0; i < b.N; i++ {
+		// generate key=sha256(i) without allocations.
+		h.Reset()
+		binary.LittleEndian.PutUint64(num[:], uint64(i))
+		h.Write(num[:])
+		key := h.Sum(keyBuf[:0])
+
+		m.Put(ctx, key)
+	}
+
+	for j := 0; j < 4; j++ {
+		for i := 0; i < b.N; i++ {
+			// generate key=sha256(i) without allocations.
+			h.Reset()
+			binary.LittleEndian.PutUint64(num[:], uint64(i))
+			h.Write(num[:])
+			key := h.Sum(keyBuf[:0])
+
+			require.True(b, m.Contains(key))
+		}
+	}
+}
+
+func TestSetPanics(t *testing.T) {
+	ctx := testlogging.Context(t)
+
+	m, err := bigmap.NewSet(ctx)
+	require.NoError(t, err)
+
+	// too short keys
+	require.Panics(t, func() { m.Put(ctx, nil) })
+	require.Panics(t, func() { m.Put(ctx, []byte{1}) })
+
+	// too long key
+	require.Panics(t, func() { m.Put(ctx, bytes.Repeat([]byte{1}, 256)) })
+}
diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go
index 6fe3d5b9..60961ed4 100644
--- a/internal/testutil/testutil.go
+++ b/internal/testutil/testutil.go
@@ -75,7 +75,7 @@ func ShouldReduceTestComplexity() bool {
 		return true
 	}
 
-	return strings.Contains(runtime.GOARCH, "arm")
+	return strings.Contains(runtime.GOARCH, "arm") && runtime.GOOS != "darwin"
 }
 
 // ShouldSkipUnicodeFilenames returns true if:
diff --git a/repo/content/index/id_test.go b/repo/content/index/id_test.go
index ee55ba79..ebcdb66c 100644
--- a/repo/content/index/id_test.go
+++ b/repo/content/index/id_test.go
@@ -2,6 +2,7 @@
 
 import (
 	"encoding/json"
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -122,6 +123,8 @@ func TestIDHash(t *testing.T) {
 			require.NoError(t, err)
 			require.Equal(t, h, cid.Hash())
 
+			require.Equal(t, fmt.Sprintf("%v%x", prefix, h), string(cid.Append(nil)))
+
 			require.Equal(t, prefix != "", cid.HasPrefix())
 			require.Equal(t, prefix, cid.Prefix())
 		}
diff --git a/repo/object/objectid_test.go b/repo/object/objectid_test.go
index 29dd916c..61560b62 100644
--- a/repo/object/objectid_test.go
+++ b/repo/object/objectid_test.go
@@ -67,6 +67,21 @@ func TestToStrings(t *testing.T) {
 	require.Equal(t, []string{"f0f0", "f1f1"}, strs)
 }
 
+func TestString(t *testing.T) {
+	cases := map[ID]string{
+		EmptyID:                  "",
+		mustParseID(t, "Dabcd"):  "abcd",
+		mustParseID(t, "abcd"):   "abcd",
+		mustParseID(t, "IIabcd"): "IIabcd",
+		mustParseID(t, "Zabcd"):  "Zabcd",
+	}
+
+	for id, str := range cases {
+		require.Equal(t, str, id.String())
+		require.Equal(t, str, string(id.Append(nil)))
+	}
+}
+
 func mustParseID(t *testing.T, s string) ID {
 	t.Helper()
 
diff --git a/snapshot/snapshotfs/snapshot_tree_walker_test.go b/snapshot/snapshotfs/snapshot_tree_walker_test.go
index 3d40b689..4f9a8529 100644
--- a/snapshot/snapshotfs/snapshot_tree_walker_test.go
+++ b/snapshot/snapshotfs/snapshot_tree_walker_test.go
@@ -19,16 +19,19 @@
 func TestSnapshotTreeWalker(t *testing.T) {
 	callbackCounter := new(int32)
 
-	w := snapshotfs.NewTreeWalker(
+	ctx, env := repotesting.NewEnvironment(t, repotesting.FormatNotImportant)
+
+	w, err := snapshotfs.NewTreeWalker(
+		ctx,
 		snapshotfs.TreeWalkerOptions{
 			EntryCallback: func(ctx context.Context, entry fs.Entry, oid object.ID, entryPath string) error {
 				atomic.AddInt32(callbackCounter, 1)
 				return nil
 			},
 		})
-	defer w.Close()
+	require.NoError(t, err)
 
-	ctx, env := repotesting.NewEnvironment(t, repotesting.FormatNotImportant)
+	defer w.Close(ctx)
 
 	sourceRoot := mockfs.NewDirectory()
 	require.Error(t, w.Process(ctx, sourceRoot, "."))
@@ -79,7 +82,10 @@ func TestSnapshotTreeWalker(t *testing.T) {
 func TestSnapshotTreeWalker_Errors(t *testing.T) {
 	someErr1 := errors.Errorf("some error")
 
-	w := snapshotfs.NewTreeWalker(
+	ctx, env := repotesting.NewEnvironment(t, repotesting.FormatNotImportant)
+
+	w, err := snapshotfs.NewTreeWalker(
+		ctx,
 		snapshotfs.TreeWalkerOptions{
 			Parallelism: 1,
 			EntryCallback: func(ctx context.Context, entry fs.Entry, oid object.ID, entryPath string) error {
@@ -90,9 +96,9 @@ func TestSnapshotTreeWalker_Errors(t *testing.T) {
 				return nil
 			},
 		})
-	defer w.Close()
+	require.NoError(t, err)
 
-	ctx, env := repotesting.NewEnvironment(t, repotesting.FormatNotImportant)
+	defer w.Close(ctx)
 
 	sourceRoot := mockfs.NewDirectory()
 	require.Error(t, w.Process(ctx, sourceRoot, "root-dir"))
@@ -118,7 +124,10 @@ func TestSnapshotTreeWalker_Errors(t *testing.T) {
 func TestSnapshotTreeWalker_MultipleErrors(t *testing.T) {
 	someErr1 := errors.Errorf("some error")
 
-	w := snapshotfs.NewTreeWalker(
+	ctx, env := repotesting.NewEnvironment(t, repotesting.FormatNotImportant)
+
+	w, err := snapshotfs.NewTreeWalker(
+		ctx,
 		snapshotfs.TreeWalkerOptions{
 			Parallelism: 1,
 			MaxErrors:   -1,
@@ -134,9 +143,9 @@ func TestSnapshotTreeWalker_MultipleErrors(t *testing.T) {
 				return nil
 			},
 		})
-	defer w.Close()
+	require.NoError(t, err)
 
-	ctx, env := repotesting.NewEnvironment(t, repotesting.FormatNotImportant)
+	defer w.Close(ctx)
 
 	sourceRoot := mockfs.NewDirectory()
 	require.Error(t, w.Process(ctx, sourceRoot, "root-dir"))
@@ -165,7 +174,10 @@ func TestSnapshotTreeWalker_MultipleErrors(t *testing.T) {
 func TestSnapshotTreeWalker_MultipleErrorsSameOID(t *testing.T) {
 	someErr1 := errors.Errorf("some error")
 
-	w := snapshotfs.NewTreeWalker(
+	ctx, env := repotesting.NewEnvironment(t, repotesting.FormatNotImportant)
+
+	w, err := snapshotfs.NewTreeWalker(
+		ctx,
 		snapshotfs.TreeWalkerOptions{
 			Parallelism: 1,
 			MaxErrors:   -1,
@@ -181,9 +193,9 @@ func TestSnapshotTreeWalker_MultipleErrorsSameOID(t *testing.T) {
 				return nil
 			},
 		})
-	defer w.Close()
+	require.NoError(t, err)
 
-	ctx, env := repotesting.NewEnvironment(t, repotesting.FormatNotImportant)
+	defer w.Close(ctx)
 
 	sourceRoot := mockfs.NewDirectory()
 	require.Error(t, w.Process(ctx, sourceRoot, "root-dir"))
diff --git a/snapshot/snapshotfs/upload_test.go b/snapshot/snapshotfs/upload_test.go
index 5fb82244..740835d9 100644
--- a/snapshot/snapshotfs/upload_test.go
+++ b/snapshot/snapshotfs/upload_test.go
@@ -909,14 +909,14 @@ func TestUpload_VirtualDirectoryWithStreamingFileWithModTime(t *testing.T) {
 			require.Equal(t, int32(1), atomic.LoadInt32(&man1.Stats.TotalDirectoryCount))
 			require.Equal(t, int32(1), atomic.LoadInt32(&man1.Stats.TotalFileCount))
 
+			// wait a little bit to ensure clock moves forward which is not always the case on Windows.
+			time.Sleep(100 * time.Millisecond)
+
 			// Rebuild tree because reader only works once.
 			staticRoot = virtualfs.NewStaticDirectory("rootdir", []fs.Entry{
 				tc.getFile(),
 			})
 
-			// wait a little bit to ensure clock moves forward which is not always the case on Windows.
-			time.Sleep(100 * time.Millisecond)
-
 			// Second upload may find some cached files depending on timestamps.
 			man2, err := u.Upload(ctx, staticRoot, policyTree, snapshot.SourceInfo{}, man1)
 			require.NoError(t, err)
diff --git a/tests/htmlui_e2e_test/htmlui_e2e_test.go b/tests/htmlui_e2e_test/htmlui_e2e_test.go
index f2846318..5b6e17d3 100644
--- a/tests/htmlui_e2e_test/htmlui_e2e_test.go
+++ b/tests/htmlui_e2e_test/htmlui_e2e_test.go
@@ -42,7 +42,7 @@ func runInBrowser(t *testing.T, run func(ctx context.Context, sp *testutil.Serve
 
 	var sp testutil.ServerParameters
 
-	_, kill := e.RunAndProcessStderr(t, sp.ProcessOutput,
+	args := []string{
 		"server", "start",
 		"--ui",
 		"--address=localhost:0",
@@ -54,8 +54,14 @@ func runInBrowser(t *testing.T, run func(ctx context.Context, sp *testutil.Serve
 		"--without-password",
 		"--override-hostname=the-hostname",
 		"--override-username=the-username",
-		// "--html="+os.Getenv("HTMLUI_BUILD_DIR"),
-	)
+	}
+
+	if h := os.Getenv("HTMLUI_BUILD_DIR"); h != "" {
+		args = append(args, "--html="+os.Getenv("HTMLUI_BUILD_DIR"))
+	}
+
+	_, kill := e.RunAndProcessStderr(t, sp.ProcessOutput, args...)
+
 	defer kill()
 
 	t.Logf("detected server parameters %#v", sp)