Repository: cockroachdb/pebble

Issue:
:  failed
. [failed](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicRace/18196671?buildTab=log) with [artifacts](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicRace/18196671?buildTab=artifacts#meta) on refs/heads/master @ [d556bba47533](https://github.com/cockroachdb/pebble/commits/d556bba47533):


```
[64 / 345] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 10s processwrapper-sandbox ... (8 actions, 7 running)
[66 / 345] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 11s processwrapper-sandbox ... (9 actions, 8 running)
[70 / 345] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 13s processwrapper-sandbox ... (8 actions, 7 running)
[71 / 345] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 14s processwrapper-sandbox ... (9 actions, 8 running)
[74 / 345] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 15s processwrapper-sandbox ... (9 actions, 8 running)
[75 / 345] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 16s processwrapper-sandbox ... (9 actions, 8 running)
[78 / 345] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 17s processwrapper-sandbox ... (8 actions, 7 running)
[81 / 345] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 19s processwrapper-sandbox ... (9 actions, 7 running)
[81 / 345] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 20s processwrapper-sandbox ... (9 actions, 8 running)
[84 / 345] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 22s processwrapper-sandbox ... (8 actions, 7 running)
[89 / 345] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 23s processwrapper-sandbox ... (9 actions, 8 running)
[95 / 345] Compiling src/google/protobuf/descriptor.cc [for tool]; 13s processwrapper-sandbox ... (8 actions running)
[99 / 345] Compiling src/google/protobuf/descriptor.cc [for tool]; 14s processwrapper-sandbox ... (9 actions, 8 running)
[102 / 345] Compiling src/google/protobuf/struct.pb.cc [for tool]; 4s processwrapper-sandbox ... (8 actions, 7 running)
[106 / 345] Compiling src/google/protobuf/text_format.cc [for tool]; 3s processwrapper-sandbox ... (8 actions, 7 running)
[108 / 345] Compiling src/google/protobuf/text_format.cc [for tool]; 4s processwrapper-sandbox ... (8 actions, 7 running)
[114 / 345] Compiling src/google/protobuf/text_format.cc [for tool]; 6s processwrapper-sandbox ... (8 actions running)
[119 / 345] GoStdlib external/io_bazel_rules_go/stdlib_/pkg [for tool]; 7s processwrapper-sandbox ... (5 actions running)
[121 / 345] GoStdlib external/io_bazel_rules_go/stdlib_/pkg [for tool]; 8s processwrapper-sandbox ... (3 actions running)
[122 / 345] GoStdlib external/io_bazel_rules_go/stdlib_/pkg [for tool]; 9s processwrapper-sandbox ... (3 actions running)
[127 / 345] GoStdlib external/io_bazel_rules_go/stdlib_/pkg [for tool]; 11s processwrapper-sandbox ... (2 actions running)
[127 / 345] GoStdlib external/io_bazel_rules_go/stdlib_/pkg [for tool]; 21s processwrapper-sandbox ... (2 actions running)
[131 / 345] GoStdlib external/io_bazel_rules_go/stdlib_/pkg; 16s processwrapper-sandbox ... (3 actions, 2 running)
[133 / 345] GoStdlib external/io_bazel_rules_go/stdlib_/pkg; 18s processwrapper-sandbox ... (2 actions, 1 running)
[137 / 345] GoStdlib external/io_bazel_rules_go/stdlib_/pkg; 19s processwrapper-sandbox ... (2 actions running)
[142 / 345] GoStdlib external/io_bazel_rules_go/stdlib_/pkg; 20s processwrapper-sandbox ... (6 actions running)
[155 / 345] GoStdlib external/io_bazel_rules_go/stdlib_/pkg; 21s processwrapper-sandbox ... (2 actions, 1 running)
[160 / 345] GoStdlib external/io_bazel_rules_go/stdlib_/pkg; 23s processwrapper-sandbox
[161 / 345] [Prepa] GoCompilePkg external/org_golang_x_sys/execabs/execabs.a ... (4 actions, 0 running)
[199 / 345] GoCompilePkg external/in_gopkg_yaml_v3/yaml_v3.a; 1s processwrapper-sandbox ... (8 actions, 7 running)
[234 / 345] GoCompilePkg external/org_golang_x_sys/unix/unix.a; 1s processwrapper-sandbox ... (8 actions, 7 running)
[255 / 345] GoCompilePkg external/com_github_gogo_protobuf/proto/proto.a; 2s processwrapper-sandbox ... (8 actions running)
[265 / 345] GoCompilePkg external/com_github_datadog_zstd/zstd.a; 2s processwrapper-sandbox ... (4 actions, 3 running)
[268 / 345] GoCompilePkg external/com_github_datadog_zstd/zstd.a; 3s processwrapper-sandbox ... (3 actions, 2 running)
[296 / 345] GoCompilePkg external/com_github_datadog_zstd/zstd.a; 4s processwrapper-sandbox ... (3 actions, 2 running)
[312 / 345] GoCompilePkg external/com_github_datadog_zstd/zstd.a; 5s processwrapper-sandbox ... (4 actions, 3 running)
[323 / 345] GoCompilePkg external/com_github_datadog_zstd/zstd.a; 6s processwrapper-sandbox ... (3 actions running)
[326 / 345] GoCompilePkg external/com_github_datadog_zstd/zstd.a; 7s processwrapper-sandbox
[327 / 345] [Prepa] GoCompilePkg external/com_github_cockroachdb_pebble/sstable/block/block.a
[331 / 345] GoCompilePkg external/com_github_cockroachdb_pebble/sstable/sstable.a; 0s processwrapper-sandbox
[332 / 345] [Prepa] GoCompilePkg external/com_github_cockroachdb_pebble/internal/manifest/manifest.a
[336 / 345] GoCompilePkg external/com_github_cockroachdb_pebble/internal/compact/compact.a; 0s processwrapper-sandbox ... (2 actions, 1 running)
[338 / 345] GoCompilePkg external/com_github_cockroachdb_pebble/pebble.a; 1s processwrapper-sandbox
[339 / 345] [Prepa] GoCompilePkg external/com_github_cockroachdb_pebble/metamorphic/metamorphic.a
[339 / 345] GoCompilePkg external/com_github_cockroachdb_pebble/metamorphic/metamorphic.a; 1s processwrapper-sandbox
[341 / 345] GoCompilePkg external/com_github_cockroachdb_pebble/internal/metamorphic/metamorphic_test.internal.a; 0s processwrapper-sandbox
[345 / 346] [Prepa] Testing @@com_github_cockroachdb_pebble//internal/metamorphic:metamorphic_test
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic:metamorphic_test; 1s processwrapper-sandbox
FAIL: @@com_github_cockroachdb_pebble//internal/metamorphic:metamorphic_test (see /home/roach/.cache/bazel/_bazel_roach/c5a4e7d36696d9cd970af2045211a7df/execroot/com_github_cockroachdb_cockroach/bazel-out/k8-fastbuild/testlogs/external/com_github_cockroachdb_pebble/internal/metamorphic/metamorphic_test/test.log)
[345 / 346] 1 / 1 tests, 1 failed
```
<details><summary>Help</summary>
<p>
</p>
</details>
<sub>

[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/bazci/githubpost/issues)

</sub>


Jira issue: PEBBLE-315

Hints from issue comments:
@cockroach-teamcity: . [failed](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicCrossversion/18503924?buildTab=log) with [artifacts](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicCrossversion/18503924?buildTab=artifacts#/) on refs/heads/master @ [ba0eeec8205a](https://github.com/cockroachdb/pebble/commits/ba0eeec8205a):


```
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15472s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15532s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15592s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15652s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15712s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15772s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15832s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15892s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15952s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16012s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16072s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16132s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16192s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16252s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16312s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16372s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16432s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16492s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16552s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16612s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16672s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16732s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16792s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16852s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16912s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16972s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17032s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17092s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17152s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17212s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17272s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17332s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17392s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17453s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17513s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17573s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17633s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17693s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17753s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17813s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17873s processwrapper-sandbox
FAIL: @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test (see /home/roach/.cache/bazel/_bazel_roach/c5a4e7d36696d9cd970af2045211a7df/execroot/com_github_cockroachdb_cockroach/bazel-out/k8-fastbuild/testlogs/external/com_github_cockroachdb_pebble/internal/metamorphic/crossversion/crossversion_test/test.log)
[344 / 345] 1 / 1 tests, 1 failed; Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17932s processwrapper-sandbox
INFO: From Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test:
INFO: Found 1 test target...
Target @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test up-to-date:
  _bazel/bin/external/com_github_cockroachdb_pebble/internal/metamorphic/crossversion/crossversion_test_/crossversion_test
INFO: Elapsed time: 17975.312s, Critical Path: 17970.94s
INFO: 222 processes: 4 internal, 218 processwrapper-sandbox.
INFO: Build completed, 1 test FAILED, 222 tot
```
<details><summary>Help</summary>
<p>

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM)
</p>
</details>
<details><summary>Same failure on other branches</summary>
<p>

- #139115 :  failed [C-test-failure O-robot T-testeng branch-release-23.2.19-rc release-blocker]
</p>
</details>
<sub>

[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/bazci/githubpost/issues)

</sub>
@jbowens: Eugh
```
Failed to publish artifacts: java.lang.OutOfMemoryError: Java heap space
```

No artifacts. I think we need to adjust the crossversion test to prune more of the artifacts as it goes. I think there are some data directories that we can clean up as we go.@jbowens: The build log has _some_ information.

```
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:99: Test directory: /artifacts/tmp/_tmp/366cdd6e86440d4e9e9c98d28c5c6592/TestMetaCrossVersion1796843652/001
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:100: Reproduction:
[05:16:48] :		 [Run the crossversion test]           SEED=1736932607645697746 FACTOR=10 ./scripts/run-crossversion-meta.sh 848f3c6eb646 19b47dc92d30 98ed66b2cad0 cd60be76d895 ba0eeec8205a
[05:16:48] :		 [Run the crossversion test]         
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:118: 0: crl-release-23.1,848f3c6eb646,/test-bin/848f3c6eb646.test (Mode = -rwxrwxrwx)
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:118: 1: crl-release-23.2,19b47dc92d30,/test-bin/19b47dc92d30.test (Mode = -rwxrwxrwx)
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:118: 2: crl-release-24.1,98ed66b2cad0,/test-bin/98ed66b2cad0.test (Mode = -rwxrwxrwx)
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:118: 3: crl-release-24.2,cd60be76d895,/test-bin/cd60be76d895.test (Mode = -rwxrwxrwx)
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:118: 4: master,ba0eeec8205a,/test-bin/ba0eeec8205a.test (Mode = -rwxrwxrwx)
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:181: Running tests with version 848f3c6e with 1 initial state(s).
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-23.1
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-23.1/#00
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-23.1/#00
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-23.1/#00
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 848f3c6e with initial state <empty>.
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:181: Running tests with version 19b47dc9 with 10 initial state(s).
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-23.2
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-023(<empty>)
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-023(<empty>)
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>)
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>)
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-005(<empty>)
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-005(<empty>)
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-015(<empty>)
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-015(<empty>)
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>)
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>)
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=random-002(<empty>)
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=random-002(<empty>)
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-022(<empty>)
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-022(<empty>)
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=random-007(<empty>)
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=random-007(<empty>)
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>)
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>)
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-023(<empty>)
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>)
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-005(<empty>)
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-015(<empty>)
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>)
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=random-007(<empty>)
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-022(<empty>)
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>)
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 19b47dc9 with initial state sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>).
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-023(<empty>)
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 19b47dc9 with initial state sha=848f3c6e-seed=16836694762171318389-opts=standard-023(<empty>).
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-015(<empty>)
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 19b47dc9 with initial state sha=848f3c6e-seed=16836694762171318389-opts=standard-015(<empty>).
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>)
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-005(<empty>)
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 19b47dc9 with initial state sha=848f3c6e-seed=16836694762171318389-opts=standard-005(<empty>).
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=random-007(<empty>)
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 19b47dc9 with initial state sha=848f3c6e-seed=16836694762171318389-opts=random-007(<empty>).
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>)
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 19b47dc9 with initial state sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>).
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>)
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 19b47dc9 with initial state sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>).
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=standard-022(<empty>)
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 19b47dc9 with initial state sha=848f3c6e-seed=16836694762171318389-opts=standard-022(<empty>).
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=random-002(<empty>)
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 19b47dc9 with initial state sha=848f3c6e-seed=16836694762171318389-opts=random-002(<empty>).
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-23.2/sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 19b47dc9 with initial state sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>).
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:181: Running tests with version 98ed66b2 with 10 initial state(s).
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.1
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=standard-027(sha=848f3c6e-seed=16836694762171318389-opts=standard-005(<empty>))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=standard-027(sha=848f3c6e-seed=16836694762171318389-opts=standard-005(<empty>))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-026(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-026(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=standard-014(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=standard-014(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=standard-026(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=standard-026(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-021(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-021(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-027(sha=848f3c6e-seed=16836694762171318389-opts=random-002(<empty>))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-027(sha=848f3c6e-seed=16836694762171318389-opts=random-002(<empty>))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=standard-001(sha=848f3c6e-seed=16836694762171318389-opts=standard-023(<empty>))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=standard-001(sha=848f3c6e-seed=16836694762171318389-opts=standard-023(<empty>))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-003(sha=848f3c6e-seed=16836694762171318389-opts=standard-022(<empty>))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-003(sha=848f3c6e-seed=16836694762171318389-opts=standard-022(<empty>))
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=standard-027(sha=848f3c6e-seed=16836694762171318389-opts=standard-005(<empty>))
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-021(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>))
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>))
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=standard-027(sha=848f3c6e-seed=16836694762171318389-opts=standard-005(<empty>))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 98ed66b2 with initial state sha=19b47dc9-seed=14563490218462585855-opts=standard-027(sha=848f3c6e-seed=16836694762171318389-opts=standard-005(<empty>)).
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 98ed66b2 with initial state sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>)).
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>))
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-021(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 98ed66b2 with initial state sha=19b47dc9-seed=14563490218462585855-opts=random-021(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)).
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=standard-026(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>))
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 98ed66b2 with initial state sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>)).
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=standard-001(sha=848f3c6e-seed=16836694762171318389-opts=standard-023(<empty>))
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=standard-026(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 98ed66b2 with initial state sha=19b47dc9-seed=14563490218462585855-opts=standard-026(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>)).
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=standard-001(sha=848f3c6e-seed=16836694762171318389-opts=standard-023(<empty>))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 98ed66b2 with initial state sha=19b47dc9-seed=14563490218462585855-opts=standard-001(sha=848f3c6e-seed=16836694762171318389-opts=standard-023(<empty>)).
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-003(sha=848f3c6e-seed=16836694762171318389-opts=standard-022(<empty>))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 98ed66b2 with initial state sha=19b47dc9-seed=14563490218462585855-opts=random-003(sha=848f3c6e-seed=16836694762171318389-opts=standard-022(<empty>)).
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=standard-014(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 98ed66b2 with initial state sha=19b47dc9-seed=14563490218462585855-opts=standard-014(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>)).
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-026(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 98ed66b2 with initial state sha=19b47dc9-seed=14563490218462585855-opts=random-026(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)).
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-24.1/sha=19b47dc9-seed=14563490218462585855-opts=random-027(sha=848f3c6e-seed=16836694762171318389-opts=random-002(<empty>))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version 98ed66b2 with initial state sha=19b47dc9-seed=14563490218462585855-opts=random-027(sha=848f3c6e-seed=16836694762171318389-opts=random-002(<empty>)).
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:181: Running tests with version cd60be76 with 10 initial state(s).
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.2
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=standard-002(sha=19b47dc9-seed=14563490218462585855-opts=standard-001(sha=848f3c6e-seed=16836694762171318389-opts=standard-023(<empty>)))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=standard-002(sha=19b47dc9-seed=14563490218462585855-opts=standard-001(sha=848f3c6e-seed=16836694762171318389-opts=standard-023(<empty>)))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=standard-029(sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>)))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=standard-029(sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>)))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-008(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>)))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-008(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>)))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-020(sha=19b47dc9-seed=14563490218462585855-opts=random-021(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-020(sha=19b47dc9-seed=14563490218462585855-opts=random-021(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-014(sha=19b47dc9-seed=14563490218462585855-opts=random-026(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-014(sha=19b47dc9-seed=14563490218462585855-opts=random-026(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=standard-014(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>)))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=standard-014(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>)))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-025(sha=19b47dc9-seed=14563490218462585855-opts=random-021(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-025(sha=19b47dc9-seed=14563490218462585855-opts=random-021(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=standard-005(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>)))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=standard-005(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>)))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=random-026(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=random-026(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=standard-006(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>)))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=standard-006(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>)))
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=standard-002(sha=19b47dc9-seed=14563490218462585855-opts=standard-001(sha=848f3c6e-seed=16836694762171318389-opts=standard-023(<empty>)))
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-025(sha=19b47dc9-seed=14563490218462585855-opts=random-021(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)))
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=random-026(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)))
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=standard-006(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>)))
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=standard-002(sha=19b47dc9-seed=14563490218462585855-opts=standard-001(sha=848f3c6e-seed=16836694762171318389-opts=standard-023(<empty>)))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version cd60be76 with initial state sha=98ed66b2-seed=3513984237228148414-opts=standard-002(sha=19b47dc9-seed=14563490218462585855-opts=standard-001(sha=848f3c6e-seed=16836694762171318389-opts=standard-023(<empty>))).
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-020(sha=19b47dc9-seed=14563490218462585855-opts=random-021(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)))
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=standard-006(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>)))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version cd60be76 with initial state sha=98ed66b2-seed=3513984237228148414-opts=standard-006(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>))).
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=standard-005(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>)))
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-025(sha=19b47dc9-seed=14563490218462585855-opts=random-021(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version cd60be76 with initial state sha=98ed66b2-seed=3513984237228148414-opts=random-025(sha=19b47dc9-seed=14563490218462585855-opts=random-021(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>))).
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-008(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>)))
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-020(sha=19b47dc9-seed=14563490218462585855-opts=random-021(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version cd60be76 with initial state sha=98ed66b2-seed=3513984237228148414-opts=random-020(sha=19b47dc9-seed=14563490218462585855-opts=random-021(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>))).
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=random-026(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version cd60be76 with initial state sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=random-026(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>))).
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=standard-014(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>)))
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-008(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>)))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version cd60be76 with initial state sha=98ed66b2-seed=3513984237228148414-opts=random-008(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>))).
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=standard-014(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>)))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version cd60be76 with initial state sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=standard-014(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>))).
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=standard-005(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>)))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version cd60be76 with initial state sha=98ed66b2-seed=3513984237228148414-opts=standard-005(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>))).
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=standard-029(sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>)))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version cd60be76 with initial state sha=98ed66b2-seed=3513984237228148414-opts=standard-029(sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>))).
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-crl-release-24.2/sha=98ed66b2-seed=3513984237228148414-opts=random-014(sha=19b47dc9-seed=14563490218462585855-opts=random-026(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version cd60be76 with initial state sha=98ed66b2-seed=3513984237228148414-opts=random-014(sha=19b47dc9-seed=14563490218462585855-opts=random-026(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>))).
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:181: Running tests with version ba0eeec8 with 10 initial state(s).
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-master
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=random-004(sha=98ed66b2-seed=3513984237228148414-opts=random-025(sha=19b47dc9-seed=14563490218462585855-opts=random-021(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>))))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=random-004(sha=98ed66b2-seed=3513984237228148414-opts=random-025(sha=19b47dc9-seed=14563490218462585855-opts=random-021(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>))))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=standard-017(sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=standard-014(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>))))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=standard-017(sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=standard-014(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>))))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=standard-023(sha=98ed66b2-seed=3513984237228148414-opts=random-008(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>))))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=standard-023(sha=98ed66b2-seed=3513984237228148414-opts=random-008(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>))))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=standard-008(sha=98ed66b2-seed=3513984237228148414-opts=standard-029(sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>))))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=standard-008(sha=98ed66b2-seed=3513984237228148414-opts=standard-029(sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>))))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=random-023(sha=98ed66b2-seed=3513984237228148414-opts=standard-002(sha=19b47dc9-seed=14563490218462585855-opts=standard-001(sha=848f3c6e-seed=16836694762171318389-opts=standard-023(<empty>))))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=random-023(sha=98ed66b2-seed=3513984237228148414-opts=standard-002(sha=19b47dc9-seed=14563490218462585855-opts=standard-001(sha=848f3c6e-seed=16836694762171318389-opts=standard-023(<empty>))))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=standard-025(sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=standard-014(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>))))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=standard-025(sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=standard-014(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>))))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=random-011(sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=random-026(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>))))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=random-011(sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=random-026(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>))))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=random-010(sha=98ed66b2-seed=3513984237228148414-opts=standard-005(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>))))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=random-010(sha=98ed66b2-seed=3513984237228148414-opts=standard-005(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>))))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=standard-002(sha=98ed66b2-seed=3513984237228148414-opts=standard-029(sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>))))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=standard-002(sha=98ed66b2-seed=3513984237228148414-opts=standard-029(sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>))))
[05:16:48] :		 [Run the crossversion test] === RUN   TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=random-011(sha=98ed66b2-seed=3513984237228148414-opts=standard-029(sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>))))
[05:16:48] :		 [Run the crossversion test] === PAUSE TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=random-011(sha=98ed66b2-seed=3513984237228148414-opts=standard-029(sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>))))
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=random-004(sha=98ed66b2-seed=3513984237228148414-opts=random-025(sha=19b47dc9-seed=14563490218462585855-opts=random-021(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>))))
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=random-011(sha=98ed66b2-seed=3513984237228148414-opts=standard-029(sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>))))
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=standard-025(sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=standard-014(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>))))
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=standard-008(sha=98ed66b2-seed=3513984237228148414-opts=standard-029(sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>))))
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=standard-023(sha=98ed66b2-seed=3513984237228148414-opts=random-008(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>))))
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=standard-002(sha=98ed66b2-seed=3513984237228148414-opts=standard-029(sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>))))
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=random-004(sha=98ed66b2-seed=3513984237228148414-opts=random-025(sha=19b47dc9-seed=14563490218462585855-opts=random-021(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>))))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version ba0eeec8 with initial state sha=cd60be76-seed=6501149822340126230-opts=random-004(sha=98ed66b2-seed=3513984237228148414-opts=random-025(sha=19b47dc9-seed=14563490218462585855-opts=random-021(sha=848f3c6e-seed=16836694762171318389-opts=random-013(<empty>)))).
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=random-011(sha=98ed66b2-seed=3513984237228148414-opts=standard-029(sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>))))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version ba0eeec8 with initial state sha=cd60be76-seed=6501149822340126230-opts=random-011(sha=98ed66b2-seed=3513984237228148414-opts=standard-029(sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>)))).
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=standard-017(sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=standard-014(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>))))
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=standard-025(sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=standard-014(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>))))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version ba0eeec8 with initial state sha=cd60be76-seed=6501149822340126230-opts=standard-025(sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=standard-014(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>)))).
[05:16:48] :		 [Run the crossversion test] === CONT  TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=random-010(sha=98ed66b2-seed=3513984237228148414-opts=standard-005(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>))))
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=standard-008(sha=98ed66b2-seed=3513984237228148414-opts=standard-029(sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>))))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version ba0eeec8 with initial state sha=cd60be76-seed=6501149822340126230-opts=standard-008(sha=98ed66b2-seed=3513984237228148414-opts=standard-029(sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>)))).
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=standard-023(sha=98ed66b2-seed=3513984237228148414-opts=random-008(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>))))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version ba0eeec8 with initial state sha=cd60be76-seed=6501149822340126230-opts=standard-023(sha=98ed66b2-seed=3513984237228148414-opts=random-008(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>)))).
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=standard-002(sha=98ed66b2-seed=3513984237228148414-opts=standard-029(sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>))))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version ba0eeec8 with initial state sha=cd60be76-seed=6501149822340126230-opts=standard-002(sha=98ed66b2-seed=3513984237228148414-opts=standard-029(sha=19b47dc9-seed=14563490218462585855-opts=random-006(sha=848f3c6e-seed=16836694762171318389-opts=standard-002(<empty>)))).
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=standard-017(sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=standard-014(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>))))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version ba0eeec8 with initial state sha=cd60be76-seed=6501149822340126230-opts=standard-017(sha=98ed66b2-seed=3513984237228148414-opts=random-013(sha=19b47dc9-seed=14563490218462585855-opts=standard-014(sha=848f3c6e-seed=16836694762171318389-opts=standard-009(<empty>)))).
[05:16:48] :		 [Run the crossversion test] === NAME  TestMetaCrossVersion/execution-master/sha=cd60be76-seed=6501149822340126230-opts=random-010(sha=98ed66b2-seed=3513984237228148414-opts=standard-005(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>))))
[05:16:48] :		 [Run the crossversion test]     crossversion_test.go:248:   Running test with version ba0eeec8 with initial state sha=cd60be76-seed=6501149822340126230-opts=random-010(sha=98ed66b2-seed=3513984237228148414-opts=standard-005(sha=19b47dc9-seed=14563490218462585855-opts=random-025(sha=848f3c6e-seed=16836694762171318389-opts=standard-001(<empty>)))).
[05:16:48] :		 [Run the crossversion test] 
[05:16:48] :		 [Run the crossversion test] 
[05:16:48] :		 [Run the crossversion test] ERROR: signal: killed
[05:16:48] :		 [Run the crossversion test] 
[05:16:48] :		 [Run the crossversion test] 62 runs completed, 1 failures, over 4h58m53s
[05:16:48] :		 [Run the crossversion test] ERROR: open /artifacts/tmp/_tmp/366cdd6e86440d4e9e9c98d28c5c6592/go-stress-shards770916666/0-62/XML_OUTPUT_FILE: no such file or directory
[05:16:48] :		 [Run the crossversion test] exit status 1
[05:16:48] :		 [Run the crossversion test] FAIL
[05:16:48] :		 [Run the crossversion test] ================================================================================
[05:16:48]W:		 [Run the crossversion test] INFO: Elapsed time: 17975.312s, Critical Path: 17970.94s
[05:16:48]W:		 [Run the crossversion test] INFO: 222 processes: 4 internal, 218 processwrapper-sandbox.
[05:16:48]W:		 [Run the crossversion test] INFO: Build completed, 1 test FAILED, 222 total actions
[05:16:48]W:		 [Run the crossversion test] INFO: 
```

I'm unclear what killed the process. Did it just timeout?@cockroach-teamcity: . [failed](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicCrossversion/18541959?buildTab=log) with [artifacts](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicCrossversion/18541959?buildTab=artifacts#/) on refs/heads/master @ [d2b08c854574](https://github.com/cockroachdb/pebble/commits/d2b08c854574):


```
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 10126s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 10186s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 10246s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 10307s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 10367s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 10427s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 10487s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 10547s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 10607s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 10667s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 10727s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 10787s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 10847s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 10907s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 10967s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11027s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11087s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11147s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11207s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11267s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11327s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11387s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11447s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11507s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11567s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11627s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11687s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11747s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11807s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11867s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11927s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11987s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12047s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12107s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12167s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12227s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12287s processwrapper-sandbox
[344 / 345] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12347s processwrapper-sandbox
FAIL: @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test (see /home/roach/.cache/bazel/_bazel_roach/c5a4e7d36696d9cd970af2045211a7df/execroot/com_github_cockroachdb_cockroach/bazel-out/k8-fastbuild/testlogs/external/com_github_cockroachdb_pebble/internal/metamorphic/crossversion/crossversion_test/test.log)
[344 / 345] 1 / 1 tests, 1 failed; Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12374s processwrapper-sandbox
INFO: From Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test:
INFO: Found 1 test target...
Target @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test up-to-date:
  _bazel/bin/external/com_github_cockroachdb_pebble/internal/metamorphic/crossversion/crossversion_test_/crossversion_test
INFO: Elapsed time: 12418.346s, Critical Path: 12413.32s
INFO: 222 processes: 4 internal, 218 processwrapper-sandbox.
INFO: Build completed, 1 test FAILED, 222 total actions
INFO: 
INFO: 
ERROR: The Build Event Protocol upload failed: Not retrying publishBuildEvents, no more attempts left: status='Status{code=UNKNOWN, description=EOF, cause=null}' UNKNOWN: UNKNOWN: E
```
<details><summary>Help</summary>
<p>

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM)
</p>
</details>
<sub>

[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/bazci/githubpost/issues)

</sub>
@cockroach-teamcity: . [failed](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicRace/18875597?buildTab=log) with [artifacts](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicRace/18875597?buildTab=artifacts#meta) on refs/heads/master @ [6949b900b3e1](https://github.com/cockroachdb/pebble/commits/6949b900b3e1):


```
[66 / 349] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 10s processwrapper-sandbox ... (9 actions, 8 running)
[71 / 349] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 12s processwrapper-sandbox ... (8 actions, 7 running)
[72 / 349] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 13s processwrapper-sandbox ... (8 actions, 7 running)
[73 / 349] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 14s processwrapper-sandbox ... (9 actions, 8 running)
[75 / 349] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 16s processwrapper-sandbox ... (9 actions, 8 running)
[77 / 349] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 17s processwrapper-sandbox ... (9 actions, 8 running)
[80 / 349] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 18s processwrapper-sandbox ... (9 actions, 8 running)
[85 / 349] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 20s processwrapper-sandbox ... (9 actions, 7 running)
[87 / 349] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 21s processwrapper-sandbox ... (8 actions, 7 running)
[89 / 349] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 22s processwrapper-sandbox ... (9 actions, 8 running)
[90 / 349] GoToolchainBinaryBuild external/go_sdk/builder [for tool]; 23s processwrapper-sandbox ... (9 actions, 8 running)
[97 / 349] Compiling src/google/protobuf/descriptor.cc [for tool]; 13s processwrapper-sandbox ... (8 actions, 7 running)
[101 / 349] Compiling src/google/protobuf/descriptor.cc [for tool]; 14s processwrapper-sandbox ... (9 actions, 7 running)
[104 / 349] Compiling src/google/protobuf/struct.pb.cc [for tool]; 4s processwrapper-sandbox ... (9 actions, 8 running)
[109 / 349] Compiling src/google/protobuf/text_format.cc [for tool]; 4s processwrapper-sandbox ... (9 actions, 7 running)
[113 / 349] Compiling src/google/protobuf/text_format.cc [for tool]; 6s processwrapper-sandbox ... (8 actions, 7 running)
[120 / 349] GoStdlib external/io_bazel_rules_go/stdlib_/pkg [for tool]; 6s processwrapper-sandbox ... (6 actions running)
[123 / 349] GoStdlib external/io_bazel_rules_go/stdlib_/pkg [for tool]; 8s processwrapper-sandbox ... (3 actions running)
[124 / 349] GoStdlib external/io_bazel_rules_go/stdlib_/pkg [for tool]; 9s processwrapper-sandbox ... (3 actions, 2 running)
[129 / 349] GoStdlib external/io_bazel_rules_go/stdlib_/pkg [for tool]; 11s processwrapper-sandbox ... (2 actions running)
[129 / 349] GoStdlib external/io_bazel_rules_go/stdlib_/pkg [for tool]; 20s processwrapper-sandbox ... (2 actions running)
[130 / 349] GoStdlib external/io_bazel_rules_go/stdlib_/pkg; 15s processwrapper-sandbox ... (4 actions, 1 running)
[134 / 349] GoStdlib external/io_bazel_rules_go/stdlib_/pkg; 17s processwrapper-sandbox ... (2 actions running)
[140 / 349] GoStdlib external/io_bazel_rules_go/stdlib_/pkg; 18s processwrapper-sandbox ... (2 actions, 1 running)
[150 / 349] GoStdlib external/io_bazel_rules_go/stdlib_/pkg; 20s processwrapper-sandbox ... (8 actions, 7 running)
[162 / 349] GoStdlib external/io_bazel_rules_go/stdlib_/pkg; 21s processwrapper-sandbox
[163 / 349] [Prepa] GoCompilePkg external/io_bazel_rules_go/go/tools/bzltestutil/chdir/chdir.a ... (3 actions, 0 running)
[196 / 349] GoCompilePkg external/in_gopkg_yaml_v3/yaml_v3.a; 1s processwrapper-sandbox ... (8 actions, 7 running)
[227 / 349] GoCompilePkg external/com_github_datadog_zstd/zstd.a; 1s processwrapper-sandbox ... (8 actions, 7 running)
[256 / 349] GoCompilePkg external/com_github_datadog_zstd/zstd.a; 2s processwrapper-sandbox ... (8 actions, 7 running)
[265 / 349] GoCompilePkg external/com_github_datadog_zstd/zstd.a; 4s processwrapper-sandbox ... (5 actions, 4 running)
[269 / 349] GoCompilePkg external/com_github_datadog_zstd/zstd.a; 5s processwrapper-sandbox ... (3 actions running)
[289 / 349] GoCompilePkg external/com_github_datadog_zstd/zstd.a; 6s processwrapper-sandbox ... (5 actions running)
[303 / 349] GoCompilePkg external/com_github_datadog_zstd/zstd.a; 7s processwrapper-sandbox ... (3 actions, 2 running)
[322 / 349] GoCompilePkg external/com_github_datadog_zstd/zstd.a; 8s processwrapper-sandbox ... (3 actions, 1 running)
[329 / 349] GoCompilePkg external/com_github_datadog_zstd/zstd.a; 10s processwrapper-sandbox
[331 / 349] [Prepa] GoCompilePkg external/com_github_cockroachdb_pebble/sstable/colblk/colblk.a
[334 / 349] GoCompilePkg external/com_github_cockroachdb_pebble/sstable/sstable.a; 1s processwrapper-sandbox
[336 / 349] GoCompilePkg external/com_github_cockroachdb_pebble/internal/manifest/manifest.a; 0s processwrapper-sandbox
[341 / 349] [Prepa] GoCompilePkg external/com_github_cockroachdb_pebble/pebble.a
[341 / 349] GoCompilePkg external/com_github_cockroachdb_pebble/pebble.a; 1s processwrapper-sandbox
[342 / 349] [Prepa] GoCompilePkg external/com_github_cockroachdb_pebble/cockroachkvs/cockroachkvs.a
[343 / 349] GoCompilePkg external/com_github_cockroachdb_pebble/metamorphic/metamorphic.a; 1s processwrapper-sandbox
[346 / 349] [Prepa] GoCompilePkgExternal external/com_github_cockroachdb_pebble/internal/metamorphic/metamorphic_test_test.external.a
[349 / 350] Testing @@com_github_cockroachdb_pebble//internal/metamorphic:metamorphic_test; 0s processwrapper-sandbox
FAIL: @@com_github_cockroachdb_pebble//internal/metamorphic:metamorphic_test (see /home/roach/.cache/bazel/_bazel_roach/c5a4e7d36696d9cd970af2045211a7df/execroot/com_github_cockroachdb_cockroach/bazel-out/k8-fastbuild/testlogs/external/com_github_cockroachdb_pebble/internal/metamorphic/metamorphic_test/test.log)
[349 / 350] 1 / 1 tests, 1 failed; Testing @@com_github_cockroachdb_pebble//internal/metamorphic:metamorphic_test; 4159s processwrapper-sandbox
INFO: Found 1 test target...
Target @@com_github_cockroachdb_pebble//internal/metamorphic:metamorphic_test up-to-date:
  _bazel/bin/external/com_github_cockroachdb_pebbl
```
<details><summary>Help</summary>
<p>
</p>
</details>
<sub>

[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/bazci/githubpost/issues)

</sub>
@cockroach-teamcity: . [failed](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicCrossversion/18906389?buildTab=log) with [artifacts](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicCrossversion/18906389?buildTab=artifacts#/) on refs/heads/master @ [6949b900b3e1](https://github.com/cockroachdb/pebble/commits/6949b900b3e1):


```
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11807s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11867s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11927s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11987s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12047s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12107s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12167s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12227s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12287s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12347s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12407s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12467s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12527s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12587s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12647s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12707s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12767s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12827s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12887s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12947s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13007s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13067s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13127s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13187s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13247s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13307s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13367s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13427s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13487s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13547s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13607s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13667s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13727s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13787s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13847s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13907s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13967s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14027s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14087s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14147s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14207s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14268s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14328s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14388s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14448s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14508s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14568s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14628s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14688s processwrapper-sandbox
[345 / 346] Testing @@com_github_cockroach
```
<details><summary>Help</summary>
<p>

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM)
</p>
</details>
<sub>

[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/bazci/githubpost/issues)

</sub>
@cockroach-teamcity: . [failed](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicCrossversion/18970873?buildTab=log) with [artifacts](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicCrossversion/18970873?buildTab=artifacts#/) on refs/heads/master @ [5c86fab0d2cd](https://github.com/cockroachdb/pebble/commits/5c86fab0d2cd):


```
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17271s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17331s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17391s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17451s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17511s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17571s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17631s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17691s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17751s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17812s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17872s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17932s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 17992s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 18052s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 18112s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 18172s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 18232s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 18292s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 18352s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 18412s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 18472s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 18532s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 18592s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 18652s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 18712s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 18772s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 18832s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 18892s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 18952s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 19012s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 19072s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 19132s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 19192s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 19252s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 19312s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 19372s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 19432s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 19492s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 19552s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 19612s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 19672s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 19732s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 19793s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 19853s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 19913s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 19973s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 20033s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 20093s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 20153s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble
```
<details><summary>Help</summary>
<p>

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM)
</p>
</details>
<sub>

[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/bazci/githubpost/issues)

</sub>
@cockroach-teamcity: . [failed](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicCrossversion/19063342?buildTab=log) with [artifacts](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicCrossversion/19063342?buildTab=artifacts#/) on refs/heads/master @ [080d15ae0c6b](https://github.com/cockroachdb/pebble/commits/080d15ae0c6b):


```
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 11988s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12048s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12108s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12168s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12228s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12288s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12348s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12408s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12468s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12528s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12588s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12648s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12708s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12768s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12828s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12888s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 12949s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13009s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13069s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13129s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13189s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13249s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13309s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13369s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13429s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13489s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13549s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13609s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13669s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13729s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13789s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13849s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13909s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13969s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14029s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14089s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14149s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14209s processwrapper-sandbox
[346 / 347] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14269s processwrapper-sandbox
FAIL: @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test (see /home/roach/.cache/bazel/_bazel_roach/c5a4e7d36696d9cd970af2045211a7df/execroot/com_github_cockroachdb_cockroach/bazel-out/k8-fastbuild/testlogs/external/com_github_cockroachdb_pebble/internal/metamorphic/crossversion/crossversion_test/test.log)
[346 / 347] 1 / 1 tests, 1 failed; Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14275s processwrapper-sandbox
INFO: From Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test:
INFO: Found 1 test target...
Target @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test up-to-date:
  _bazel/bin/external/com_github_cockroachdb_pebble/internal/metamorphic/crossversion/crossversion_test_/crossversion_test
INFO: Elapsed time: 14318.931s, Critical Path: 14315.24s
INFO: 224 processes: 4 internal, 220 processwrapper-sandbox.
INFO: Build completed, 1 test FAILED, 224 total actions
INFO: 
IN
```
<details><summary>Help</summary>
<p>

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM)
</p>
</details>
<sub>

[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/bazci/githubpost/issues)

</sub>
@cockroach-teamcity: . [failed](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicTwoInstance/19087036?buildTab=log) with [artifacts](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicTwoInstance/19087036?buildTab=artifacts#meta) on refs/heads/master @ [a48d4fcc0d3a](https://github.com/cockroachdb/pebble/commits/a48d4fcc0d3a):


```
stdout: 
, stderr: 
```
<details><summary>Help</summary>
<p>
</p>
</details>
<sub>

[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/bazci/githubpost/issues)

</sub>
@cockroach-teamcity: . [failed](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_Pebble_Metamorphic/19087034?buildTab=log) with [artifacts](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_Pebble_Metamorphic/19087034?buildTab=artifacts#meta) on refs/heads/master @ [a48d4fcc0d3a](https://github.com/cockroachdb/pebble/commits/a48d4fcc0d3a):


```
stdout: 
, stderr: 
```
<details><summary>Help</summary>
<p>
</p>
</details>
<sub>

[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/bazci/githubpost/issues)

</sub>
@cockroach-teamcity: . [failed](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicRace/19087035?buildTab=log) with [artifacts](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicRace/19087035?buildTab=artifacts#meta) on refs/heads/master @ [a48d4fcc0d3a](https://github.com/cockroachdb/pebble/commits/a48d4fcc0d3a):


```
stdout: 
, stderr: 
```
<details><summary>Help</summary>
<p>
</p>
</details>
<sub>

[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/bazci/githubpost/issues)

</sub>
@cockroach-teamcity: . [failed](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicTwoInstance/19099886?buildTab=log) with [artifacts](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicTwoInstance/19099886?buildTab=artifacts#meta) on refs/heads/master @ [7a210e2f2a9d](https://github.com/cockroachdb/pebble/commits/7a210e2f2a9d):


```
stdout: 
, stderr: 
```
<details><summary>Help</summary>
<p>
</p>
</details>
<sub>

[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/bazci/githubpost/issues)

</sub>
@cockroach-teamcity: . [failed](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_Pebble_Metamorphic/19099884?buildTab=log) with [artifacts](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_Pebble_Metamorphic/19099884?buildTab=artifacts#meta) on refs/heads/master @ [7a210e2f2a9d](https://github.com/cockroachdb/pebble/commits/7a210e2f2a9d):


```
stdout: 
, stderr: 
```
<details><summary>Help</summary>
<p>
</p>
</details>
<sub>

[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/bazci/githubpost/issues)

</sub>
@cockroach-teamcity: . [failed](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicRace/19099885?buildTab=log) with [artifacts](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicRace/19099885?buildTab=artifacts#meta) on refs/heads/master @ [7a210e2f2a9d](https://github.com/cockroachdb/pebble/commits/7a210e2f2a9d):


```
stdout: 
, stderr: 
```
<details><summary>Help</summary>
<p>
</p>
</details>
<sub>

[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/bazci/githubpost/issues)

</sub>
@jbowens: Recent failures are build errors that will be resolved by cockroachdb/cockroach#142769.@cockroach-teamcity: . [failed](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicCrossversion/19329371?buildTab=log) with [artifacts](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicCrossversion/19329371?buildTab=artifacts#/) on refs/heads/master @ [de903d98fbe6](https://github.com/cockroachdb/pebble/commits/de903d98fbe6):


```
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 2863s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 2923s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 2983s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 3043s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 3103s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 3163s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 3223s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 3283s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 3343s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 3403s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 3463s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 3523s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 3583s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 3643s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 3703s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 3763s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 3823s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 3883s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 3943s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4003s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4063s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4123s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4183s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4243s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4303s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4363s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4423s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4483s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4543s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4603s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4664s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4724s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4784s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4844s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4904s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4964s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5024s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5084s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5144s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5204s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5264s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5324s processwrapper-sandbox
FAIL: @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test (see /home/roach/.cache/bazel/_bazel_roach/c5a4e7d36696d9cd970af2045211a7df/execroot/com_github_cockroachdb_cockroach/bazel-out/k8-fastbuild/testlogs/external/com_github_cockroachdb_pebble/internal/metamorphic/crossversion/crossversion_test/test.log)
[348 / 349] 1 / 1 tests, 1 failed; Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5355s processwrapper-sandbox
INFO: From Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test:
INFO: Found 1 test target...
Target @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test up-to-date:
  _bazel/bin/external/com_github_cockroachdb_pebble/internal/metamorphic/crossversion/crossversion_test_/crossversion_test
INFO: Elapsed time: 5401.595s, Critical Path: 5398.04s
INFO: 226 processes: 4 internal, 222 processwrapper-sandbox.
```
<details><summary>Help</summary>
<p>

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM)
</p>
</details>
<details><summary>Same failure on other branches</summary>
<p>

- #143948 :  failed [C-test-failure O-robot T-testeng branch-release-23.2 release-blocker]
</p>
</details>
<sub>

[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/bazci/githubpost/issues)

</sub>
@cockroach-teamcity: . [failed](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicCrossversion/19334474?buildTab=log) with [artifacts](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicCrossversion/19334474?buildTab=artifacts#/) on refs/heads/master @ [de903d98fbe6](https://github.com/cockroachdb/pebble/commits/de903d98fbe6):


```
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13668s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13728s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13788s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13848s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13908s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 13968s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14028s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14088s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14148s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14208s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14268s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14328s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14388s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14448s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14508s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14568s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14629s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14689s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14749s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14809s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14869s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14929s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 14989s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15049s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15109s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15169s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15229s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15289s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15349s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15409s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15469s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15529s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15589s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15649s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15709s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15769s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15829s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15889s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 15949s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16009s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16069s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16129s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16189s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16249s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16309s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16369s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16429s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16490s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 16550s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metam
```
<details><summary>Help</summary>
<p>

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM)
</p>
</details>
<sub>

[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/bazci/githubpost/issues)

</sub>
@cockroach-teamcity: . [failed](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicCrossversion/19368023?buildTab=log) with [artifacts](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicCrossversion/19368023?buildTab=artifacts#/) on refs/heads/master @ [567a8d3e3a3a](https://github.com/cockroachdb/pebble/commits/567a8d3e3a3a):


```
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4664s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4724s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4784s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4844s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4904s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 4964s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5025s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5085s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5145s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5205s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5265s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5325s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5385s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5445s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5505s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5565s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5625s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5685s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5745s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5805s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5865s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5925s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 5985s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6045s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6105s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6165s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6225s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6285s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6345s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6405s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6465s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6525s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6585s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6645s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6705s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6765s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6825s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6885s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6945s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7005s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7066s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7126s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7186s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7246s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7306s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7366s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7426s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7486s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7546s processwrapper-sandbox
[348 / 349] Testing @@com_github_cockroachdb_pebble//internal/metamorphi
```
<details><summary>Help</summary>
<p>

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM)
</p>
</details>
<details><summary>Same failure on other branches</summary>
<p>

- #144128 :  failed [C-test-failure O-robot T-testeng branch-release-23.2 release-blocker]
</p>
</details>
<sub>

[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/bazci/githubpost/issues)

</sub>
@cockroach-teamcity: . [failed](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicCrossversion/19404146?buildTab=log) with [artifacts](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_NightlyPebbleMetamorphicCrossversion/19404146?buildTab=artifacts#/) on refs/heads/master @ [d343bc4bbeaa](https://github.com/cockroachdb/pebble/commits/d343bc4bbeaa):


```
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6406s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6466s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6526s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6586s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6646s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6706s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6766s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6826s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6886s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 6946s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7006s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7066s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7126s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7186s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7246s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7306s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7366s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7426s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7486s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7546s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7606s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7666s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7726s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7786s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7846s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7906s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 7966s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 8026s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 8086s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 8146s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 8206s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 8266s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 8326s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 8386s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 8446s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 8506s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 8566s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 8626s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 8687s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 8747s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 8807s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 8867s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 8927s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 8987s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 9047s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 9107s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 9167s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 9227s processwrapper-sandbox
[353 / 354] Testing @@com_github_cockroachdb_pebble//internal/metamorphic/crossversion:crossversion_test; 9287s processwrapper-sandbox
[353 / 354] Testing @@com_github_
```
<details><summary>Help</summary>
<p>

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM)
</p>
</details>
<sub>

[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/bazci/githubpost/issues)

</sub>


Solution patch:
diff --git a/checkpoint.go b/checkpoint.go
index 5f599bd6..cef9b8f1 100644
--- a/checkpoint.go
+++ b/checkpoint.go
@@ -232,7 +232,7 @@ func (d *DB) Checkpoint(
 
 	{
 		// Copy the OPTIONS.
-		srcPath := base.MakeFilepath(fs, d.dirname, fileTypeOptions, optionsFileNum)
+		srcPath := base.MakeFilepath(fs, d.dirname, base.FileTypeOptions, optionsFileNum)
 		destPath := fs.PathJoin(destDir, fs.PathBase(srcPath))
 		ckErr = copyCheckpointOptions(fs, srcPath, destPath)
 		if ckErr != nil {
@@ -289,7 +289,7 @@ func (d *DB) Checkpoint(
 				}
 				requiredVirtualBackingFiles[fileBacking.DiskFileNum] = struct{}{}
 			}
-			meta, err := d.objProvider.Lookup(fileTypeTable, fileBacking.DiskFileNum)
+			meta, err := d.objProvider.Lookup(base.FileTypeTable, fileBacking.DiskFileNum)
 			if err != nil {
 				ckErr = err
 				return ckErr
@@ -305,7 +305,7 @@ func (d *DB) Checkpoint(
 				continue
 			}
 
-			srcPath := base.MakeFilepath(fs, d.dirname, fileTypeTable, fileBacking.DiskFileNum)
+			srcPath := base.MakeFilepath(fs, d.dirname, base.FileTypeTable, fileBacking.DiskFileNum)
 			destPath := fs.PathJoin(destDir, fs.PathBase(srcPath))
 			ckErr = vfs.LinkOrCopy(fs, srcPath, destPath)
 			if ckErr != nil {
@@ -331,7 +331,7 @@ func (d *DB) Checkpoint(
 		return ckErr
 	}
 	if len(remoteFiles) > 0 {
-		ckErr = d.objProvider.CheckpointState(fs, destDir, fileTypeTable, remoteFiles)
+		ckErr = d.objProvider.CheckpointState(fs, destDir, base.FileTypeTable, remoteFiles)
 		if ckErr != nil {
 			return ckErr
 		}
@@ -439,7 +439,7 @@ func (d *DB) writeCheckpointManifest(
 	// If some files are excluded from the checkpoint, also append a block that
 	// records those files as deleted.
 	if err := func() error {
-		srcPath := base.MakeFilepath(fs, d.dirname, fileTypeManifest, manifestFileNum)
+		srcPath := base.MakeFilepath(fs, d.dirname, base.FileTypeManifest, manifestFileNum)
 		destPath := fs.PathJoin(destDirPath, fs.PathBase(srcPath))
 		src, err := fs.Open(srcPath, vfs.SequentialReadsOption)
 		if err != nil {
@@ -505,7 +505,7 @@ func (d *DB) writeCheckpointManifest(
 	if err != nil {
 		return err
 	}
-	if err := manifestMarker.Move(base.MakeFilename(fileTypeManifest, manifestFileNum)); err != nil {
+	if err := manifestMarker.Move(base.MakeFilename(base.FileTypeManifest, manifestFileNum)); err != nil {
 		return err
 	}
 	return manifestMarker.Close()
diff --git a/cmd/pebble/db.go b/cmd/pebble/db.go
index 939c81be..6cae7eda 100644
--- a/cmd/pebble/db.go
+++ b/cmd/pebble/db.go
@@ -55,10 +55,8 @@ type pebbleDB struct {
 }
 
 func newPebbleDB(dir string) DB {
-	cache := pebble.NewCache(cacheSize)
-	defer cache.Unref()
 	opts := &pebble.Options{
-		Cache:                       cache,
+		CacheSize:                   cacheSize,
 		Comparer:                    &cockroachkvs.Comparer,
 		DisableWAL:                  disableWAL,
 		FormatMajorVersion:          pebble.FormatNewest,
diff --git a/cmd/pebble/replay.go b/cmd/pebble/replay.go
index f927a0d7..5b6b8bab 100644
--- a/cmd/pebble/replay.go
+++ b/cmd/pebble/replay.go
@@ -24,7 +24,6 @@ import (
 	"github.com/cockroachdb/pebble/bloom"
 	"github.com/cockroachdb/pebble/cockroachkvs"
 	"github.com/cockroachdb/pebble/internal/base"
-	"github.com/cockroachdb/pebble/internal/cache"
 	"github.com/cockroachdb/pebble/replay"
 	"github.com/cockroachdb/pebble/vfs"
 	"github.com/spf13/cobra"
@@ -297,12 +296,6 @@ func (c *replayConfig) getCheckpointDir(r *replay.Runner) string {
 
 func (c *replayConfig) parseHooks() *pebble.ParseHooks {
 	return &pebble.ParseHooks{
-		NewCache: func(size int64) *cache.Cache {
-			if c.maxCacheSize != 0 && size > c.maxCacheSize {
-				size = c.maxCacheSize
-			}
-			return cache.New(size)
-		},
 		NewComparer: makeComparer,
 		NewFilterPolicy: func(name string) (pebble.FilterPolicy, error) {
 			switch name {
@@ -355,7 +348,13 @@ func (c *replayConfig) parseCustomOptions(optsStr string, opts *pebble.Options)
 			value = strings.TrimSpace(value[i+1:])
 		}
 	}
-	return opts.Parse(buf.String(), c.parseHooks())
+	if err := opts.Parse(buf.String(), c.parseHooks()); err != nil {
+		return err
+	}
+	if c.maxCacheSize != 0 && opts.CacheSize > c.maxCacheSize {
+		opts.CacheSize = c.maxCacheSize
+	}
+	return nil
 }
 
 func (c *replayConfig) cleanUp() error {
diff --git a/cockroachkvs/blockproperties.go b/cockroachkvs/blockproperties.go
index 22ebd0b9..d5bc61b1 100644
--- a/cockroachkvs/blockproperties.go
+++ b/cockroachkvs/blockproperties.go
@@ -45,6 +45,14 @@ type MVCCWallTimeIntervalRangeKeyMask struct {
 	sstable.BlockIntervalFilter
 }
 
+// Init initializees the mask and its block interval filter.
+func (m *MVCCWallTimeIntervalRangeKeyMask) Init() {
+	m.BlockIntervalFilter.Init(
+		mvccWallTimeIntervalCollector,
+		0, math.MaxUint64,
+		MVCCBlockIntervalSuffixReplacer{})
+}
+
 // SetSuffix implements the pebble.BlockPropertyFilterMask interface.
 func (m *MVCCWallTimeIntervalRangeKeyMask) SetSuffix(suffix []byte) error {
 	if len(suffix) == 0 {
diff --git a/cockroachkvs/cockroachkvs.go b/cockroachkvs/cockroachkvs.go
index 3bdae183..5ac11813 100644
--- a/cockroachkvs/cockroachkvs.go
+++ b/cockroachkvs/cockroachkvs.go
@@ -70,6 +70,9 @@ var Comparer = base.Comparer{
 	FormatKey: FormatKey,
 
 	Separator: func(dst, a, b []byte) []byte {
+		if len(a) == 0 || len(b) == 0 {
+			panic(errors.AssertionFailedf("cannot separate empty keys"))
+		}
 		aKey, ok := getKeyPartFromEngineKey(a)
 		if !ok {
 			return append(dst, a...)
@@ -78,8 +81,8 @@ var Comparer = base.Comparer{
 		if !ok {
 			return append(dst, a...)
 		}
-		// If the keys are the same just return a.
-		if bytes.Equal(aKey, bKey) {
+		// If the keys are the same or one is empty, just return a.
+		if bytes.Equal(aKey, bKey) || len(aKey) == 0 || len(bKey) == 0 {
 			return append(dst, a...)
 		}
 		n := len(dst)
@@ -94,6 +97,13 @@ var Comparer = base.Comparer{
 	},
 
 	Successor: func(dst, a []byte) []byte {
+		// If a is empty, the key consisting of just a single 0x00 byte is the
+		// smallest non-empty key ≥ a. We don't return an empty key because
+		// empty keys are problematic when combined with a synthetic prefix
+		// (they form a key that has no trailing sentinel byte).
+		if len(a) == 0 {
+			return append(dst, 0x00)
+		}
 		aKey, ok := getKeyPartFromEngineKey(a)
 		if !ok {
 			return append(dst, a...)
@@ -122,6 +132,8 @@ var Comparer = base.Comparer{
 		return append(append(dst, a...), 0)
 	},
 
+	ValidateKey: validateEngineKey,
+
 	Name: "cockroach_comparator",
 }
 
@@ -184,6 +196,26 @@ func EncodeTimestamp(key []byte, walltime uint64, logical uint32) []byte {
 	return key
 }
 
+// NewTimestampSuffix allocates and encodes a new suffix byte slice encoding the
+// provided timestamp.
+func NewTimestampSuffix(wallTime uint64, logical uint32) []byte {
+	switch {
+	case wallTime == 0 && logical == 0:
+		return nil
+	case logical == 0:
+		v := make([]byte, suffixLenWithWall)
+		binary.BigEndian.PutUint64(v, wallTime)
+		v[suffixLenWithWall-1] = suffixLenWithWall
+		return v
+	default:
+		v := make([]byte, suffixLenWithLogical)
+		binary.BigEndian.PutUint64(v, wallTime)
+		binary.BigEndian.PutUint32(v[8:], logical)
+		v[suffixLenWithLogical-1] = suffixLenWithLogical
+		return v
+	}
+}
+
 // DecodeMVCCTimestampSuffix decodes an MVCC timestamp from its Pebble representation,
 // including the length suffix.
 func DecodeMVCCTimestampSuffix(encodedTS []byte) (wallTime uint64, logical uint32, err error) {
@@ -266,7 +298,7 @@ func Split(key []byte) int {
 		return 0
 	}
 	if invariants.Enabled {
-		checkEngineKey(key)
+		validateEngineKey.MustValidate(key)
 	}
 	// Last byte is the version length + 1 when there is a version, else it is
 	// 0.
@@ -281,8 +313,8 @@ func Compare(a, b []byte) int {
 		return cmp.Compare(len(a), len(b))
 	}
 	if invariants.Enabled {
-		checkEngineKey(a)
-		checkEngineKey(b)
+		validateEngineKey.MustValidate(a)
+		validateEngineKey.MustValidate(b)
 	}
 	aSuffixLen := int(a[len(a)-1])
 	aSuffixStart := len(a) - aSuffixLen
@@ -343,8 +375,8 @@ func Equal(a, b []byte) bool {
 		return len(a) == len(b)
 	}
 	if invariants.Enabled {
-		checkEngineKey(a)
-		checkEngineKey(b)
+		validateEngineKey.MustValidate(a)
+		validateEngineKey.MustValidate(b)
 	}
 	aSuffixLen := int(a[len(a)-1])
 	aSuffixStart := len(a) - aSuffixLen
@@ -743,34 +775,60 @@ func (ks *cockroachKeySeeker) IsLowerBound(k []byte, syntheticSuffix []byte) boo
 	if len(syntheticSuffix) > 0 {
 		return ComparePointSuffixes(syntheticSuffix, k[len(roachKey)+1:]) >= 0
 	}
+	firstRowWall := ks.mvccWallTimes.At(0)
+	firstLogical := ks.mvccLogical.At(0)
+	// If the first row's WallTime and LogicalTime are both zero, the first row
+	// is either (a) unversioned (and sorts before all other suffixes) or (b) is
+	// a non-MVCC key with an untyped version.
+	if firstRowWall == 0 && firstLogical == 0 {
+		return ComparePointSuffixes(ks.untypedVersions.At(0), version) >= 0
+	}
+
 	var wallTime uint64
 	var logicalTime uint32
 	switch len(version) {
 	case engineKeyNoVersion:
+		// The zero-length version is the smallest possible suffix.
+		return true
 	case engineKeyVersionWallTimeLen:
 		wallTime = binary.BigEndian.Uint64(version[:8])
 	case engineKeyVersionWallAndLogicalTimeLen, engineKeyVersionWallLogicalAndSyntheticTimeLen:
 		wallTime = binary.BigEndian.Uint64(version[:8])
 		logicalTime = binary.BigEndian.Uint32(version[8:12])
 	default:
-		// The provided key `k` is not a MVCC key. Assert that the first key in
-		// the block is also not an MVCC key. If it were, that would mean there
-		// exists both a MVCC key and a non-MVCC key with the same prefix.
+		// The provided key `k` is not a MVCC key.
 		//
-		// TODO(jackson): Double check that we'll never produce index separators
-		// that are invalid version lengths.
-		if invariants.Enabled && ks.mvccWallTimes.At(0) != 0 {
-			panic("comparing timestamp with untyped suffix")
+		// The first row IS an MVCC key. We don't expect this to happen in
+		// practice, so this path is not performance sensitive. We reconsistute
+		// the first row's MVCC suffix and then compare it to the provided key's
+		// non-MVCC suffix.
+
+		//gcassert:noescape
+		var buf [13]byte
+		//gcassert:inline
+		binary.BigEndian.PutUint64(buf[:], firstRowWall)
+		var firstRowSuffix []byte
+		if firstLogical == 0 {
+			buf[8] = 9
+			firstRowSuffix = buf[:9]
+		} else {
+			//gcassert:inline
+			binary.BigEndian.PutUint32(buf[8:], uint32(firstLogical))
+			buf[12] = 13
+			firstRowSuffix = buf[:13]
 		}
-		return ComparePointSuffixes(ks.untypedVersions.At(0), version) >= 0
+		return ComparePointSuffixes(firstRowSuffix, version) >= 0
 	}
 
 	// NB: The sign comparison is inverted because suffixes are sorted such that
 	// the largest timestamps are "smaller" in the lexicographical ordering.
-	if v := cmp.Compare(ks.mvccWallTimes.At(0), wallTime); v != 0 {
+	if v := cmp.Compare(firstRowWall, wallTime); v != 0 {
+		// The empty-suffixed zero-timestamped key sorts first. If the first row
+		// has zero wall and logical times, it sorts before k and k is not a
+		// lower bound.
 		return v < 0
 	}
-	return cmp.Compare(uint32(ks.mvccLogical.At(0)), logicalTime) <= 0
+	return cmp.Compare(uint32(firstLogical), logicalTime) <= 0
 }
 
 // SeekGE is part of the KeySeeker interface.
@@ -983,16 +1041,20 @@ func (ks *cockroachKeySeeker) MaterializeUserKeyWithSyntheticSuffix(
 //go:linkname memmove runtime.memmove
 func memmove(to, from unsafe.Pointer, n uintptr)
 
-func checkEngineKey(k []byte) {
+var validateEngineKey base.ValidateKey = func(k []byte) error {
 	if len(k) == 0 {
-		panic(errors.AssertionFailedf("empty key"))
+		// It would be nice to assert that the key is not empty, but
+		// unfortunately separator keys within index blocks may be empty in the
+		// case that a sstable (and its single data block) has no keys.
+		return nil
 	}
 	if int(k[len(k)-1]) >= len(k) {
-		panic(errors.AssertionFailedf("malformed key terminator byte: %x", k))
+		return errors.AssertionFailedf("malformed key terminator byte: %x", k)
 	}
 	if k[len(k)-1] == 1 {
-		panic(errors.AssertionFailedf("invalid key terminator byte 1"))
+		return errors.AssertionFailedf("invalid key terminator byte 1 in key: %x", k)
 	}
+	return nil
 }
 
 // FormatKey returns a formatter for the user key.
@@ -1003,6 +1065,10 @@ func FormatKey(key []byte) fmt.Formatter {
 type formatKey []byte
 
 func (fk formatKey) Format(f fmt.State, c rune) {
+	if len(fk) == 0 {
+		fmt.Fprint(f, "<empty>")
+		return
+	}
 	i := Split(fk)
 	fmt.Fprintf(f, "%s", []byte(fk)[:i-1])
 	if i == len(fk) {
@@ -1013,6 +1079,11 @@ func (fk formatKey) Format(f fmt.State, c rune) {
 	formatKeySuffix(fk[i:]).Format(f, c)
 }
 
+// FormatKeySuffix returns a formatter for the user key suffix.
+func FormatKeySuffix(suffix []byte) fmt.Formatter {
+	return formatKeySuffix(suffix)
+}
+
 type formatKeySuffix []byte
 
 func (fk formatKeySuffix) Format(f fmt.State, c rune) {
@@ -1051,7 +1122,15 @@ func (fk formatKeySuffix) Format(f fmt.State, c rune) {
 //     c. <WALLTIME-SECONDS>,<LOGICAL>
 //     d. <WALLTIME-SECONDS>.<WALLTIME-NANOS>,<LOGICAL>
 //  2. A lock table key in the format: <STRENGTH-BYTE>,<TXNUUID>
+//  3. A raw hex-encoded string prefixed with "hex:".
 func ParseFormattedKey(formattedKey string) []byte {
+	if strings.HasPrefix(formattedKey, "hex:") {
+		decoded, err := hex.DecodeString(formattedKey[4:])
+		if err != nil {
+			panic(errors.Newf("invalid hex-encoded key: %w", err))
+		}
+		return decoded
+	}
 	i := strings.IndexByte(formattedKey, '@')
 	if i == -1 {
 		// Suffix-less.
diff --git a/compaction.go b/compaction.go
index 780283f2..708a20f0 100644
--- a/compaction.go
+++ b/compaction.go
@@ -1500,7 +1500,7 @@ func (d *DB) flush1() (bytesFlushed uint64, err error) {
 		Err:        err,
 	}
 	if err == nil {
-		validateVersionEdit(ve, d.opts.Experimental.KeyValidationFunc, d.opts.Comparer.FormatKey, d.opts.Logger)
+		validateVersionEdit(ve, d.opts.Comparer.ValidateKey, d.opts.Comparer.FormatKey, d.opts.Logger)
 		for i := range ve.NewFiles {
 			e := &ve.NewFiles[i]
 			info.Output = append(info.Output, e.Meta.TableInfo())
@@ -2320,7 +2320,7 @@ func (d *DB) cleanupVersionEdit(ve *versionEdit) {
 		// Add this file to zombie tables as well, as the versionSet
 		// asserts on whether every obsolete file was at one point
 		// marked zombie.
-		d.mu.versions.zombieTables[obsoleteFiles[i].DiskFileNum] = tableInfo{
+		d.mu.versions.zombieTables[obsoleteFiles[i].DiskFileNum] = objectInfo{
 			fileInfo: fileInfo{
 				FileNum:  obsoleteFiles[i].DiskFileNum,
 				FileSize: obsoleteFiles[i].Size,
@@ -2356,7 +2356,7 @@ func (d *DB) compact1(c *compaction, errChannel chan error) (err error) {
 
 	info.Duration = d.timeNow().Sub(startTime)
 	if err == nil {
-		validateVersionEdit(ve, d.opts.Experimental.KeyValidationFunc, d.opts.Comparer.FormatKey, d.opts.Logger)
+		validateVersionEdit(ve, d.opts.Comparer.ValidateKey, d.opts.Comparer.FormatKey, d.opts.Logger)
 		err = func() error {
 			var err error
 			d.mu.versions.logLock()
@@ -2450,7 +2450,7 @@ func (d *DB) runCopyCompaction(
 		},
 	}
 
-	objMeta, err := d.objProvider.Lookup(fileTypeTable, inputMeta.FileBacking.DiskFileNum)
+	objMeta, err := d.objProvider.Lookup(base.FileTypeTable, inputMeta.FileBacking.DiskFileNum)
 	if err != nil {
 		return nil, compact.Stats{}, err
 	}
@@ -2512,7 +2512,7 @@ func (d *DB) runCopyCompaction(
 	deleteOnExit := false
 	defer func() {
 		if deleteOnExit {
-			_ = d.objProvider.Remove(fileTypeTable, newMeta.FileBacking.DiskFileNum)
+			_ = d.objProvider.Remove(base.FileTypeTable, newMeta.FileBacking.DiskFileNum)
 		}
 	}()
 
@@ -2520,7 +2520,7 @@ func (d *DB) runCopyCompaction(
 	if objMeta.IsExternal() {
 		ctx := context.TODO()
 		src, err := d.objProvider.OpenForReading(
-			ctx, fileTypeTable, inputMeta.FileBacking.DiskFileNum, objstorage.OpenOptions{},
+			ctx, base.FileTypeTable, inputMeta.FileBacking.DiskFileNum, objstorage.OpenOptions{},
 		)
 		if err != nil {
 			return nil, compact.Stats{}, err
@@ -2532,7 +2532,7 @@ func (d *DB) runCopyCompaction(
 		}()
 
 		w, _, err := d.objProvider.Create(
-			ctx, fileTypeTable, newMeta.FileBacking.DiskFileNum,
+			ctx, base.FileTypeTable, newMeta.FileBacking.DiskFileNum,
 			objstorage.CreateOptions{
 				PreferSharedStorage: remote.ShouldCreateShared(d.opts.Experimental.CreateOnShared, c.outputLevel.level),
 			},
@@ -2586,7 +2586,7 @@ func (d *DB) runCopyCompaction(
 		newMeta.Size = wrote
 	} else {
 		_, err := d.objProvider.LinkOrCopyFromLocal(context.TODO(), d.opts.FS,
-			d.objProvider.Path(objMeta), fileTypeTable, newMeta.FileBacking.DiskFileNum,
+			d.objProvider.Path(objMeta), base.FileTypeTable, newMeta.FileBacking.DiskFileNum,
 			objstorage.CreateOptions{PreferSharedStorage: true})
 		if err != nil {
 			return nil, compact.Stats{}, err
@@ -2936,7 +2936,7 @@ func (d *DB) runCompaction(
 			// Add this file to zombie tables as well, as the versionSet
 			// asserts on whether every obsolete file was at one point
 			// marked zombie.
-			d.mu.versions.zombieTables[backing.DiskFileNum] = tableInfo{
+			d.mu.versions.zombieTables[backing.DiskFileNum] = objectInfo{
 				fileInfo: fileInfo{
 					FileNum:  backing.DiskFileNum,
 					FileSize: backing.Size,
@@ -3197,7 +3197,7 @@ func (d *DB) newCompactionOutput(
 		PreferSharedStorage: remote.ShouldCreateShared(d.opts.Experimental.CreateOnShared, c.outputLevel.level),
 		WriteCategory:       writeCategory,
 	}
-	writable, objMeta, err := d.objProvider.Create(ctx, fileTypeTable, diskFileNum, createOpts)
+	writable, objMeta, err := d.objProvider.Create(ctx, base.FileTypeTable, diskFileNum, createOpts)
 	if err != nil {
 		return objstorage.ObjectMetadata{}, nil, nil, err
 	}
@@ -3218,9 +3218,8 @@ func (d *DB) newCompactionOutput(
 
 	writerOpts.SetInternal(sstableinternal.WriterOptions{
 		CacheOpts: sstableinternal.CacheOptions{
-			Cache:   d.opts.Cache,
-			CacheID: d.cacheID,
-			FileNum: diskFileNum,
+			CacheHandle: d.cacheHandle,
+			FileNum:     diskFileNum,
 		},
 	})
 
@@ -3241,10 +3240,10 @@ func (d *DB) newCompactionOutput(
 // validateVersionEdit validates that start and end keys across new and deleted
 // files in a versionEdit pass the given validation function.
 func validateVersionEdit(
-	ve *versionEdit, validateFn func([]byte) error, format base.FormatKey, logger Logger,
+	ve *versionEdit, vk base.ValidateKey, format base.FormatKey, logger Logger,
 ) {
 	validateKey := func(f *manifest.FileMetadata, key []byte) {
-		if err := validateFn(key); err != nil {
+		if err := vk.Validate(key); err != nil {
 			logger.Fatalf("pebble: version edit validation failed (key=%s file=%s): %v", format(key), f, err)
 		}
 	}
diff --git a/comparer.go b/comparer.go
index 5f31d240..542499e1 100644
--- a/comparer.go
+++ b/comparer.go
@@ -30,9 +30,5 @@ type Comparer = base.Comparer
 // DefaultComparer exports the base.DefaultComparer variable.
 var DefaultComparer = base.DefaultComparer
 
-// CheckComparer is a mini test suite that verifies a comparer implementation.
-//
-// It takes strictly ordered (according to the comparator) lists of prefixes and
-// suffixes. Both lists must contain the empty slice. It is recommended that
-// both lists have at least three elements.
+// CheckComparer exports the base.CheckComparer type.
 var CheckComparer = base.CheckComparer
diff --git a/db.go b/db.go
index 03368cc7..bb2010df 100644
--- a/db.go
+++ b/db.go
@@ -283,7 +283,7 @@ type DB struct {
 	diskAvailBytes       atomic.Uint64
 	lowDiskSpaceReporter lowDiskSpaceReporter
 
-	cacheID        cache.ID
+	cacheHandle    *cache.Handle
 	dirname        string
 	opts           *Options
 	cmp            Compare
@@ -305,7 +305,7 @@ type DB struct {
 	fileLock *Lock
 	dataDir  vfs.File
 
-	fileCache            *fileCacheContainer
+	fileCache            *fileCacheHandle
 	newIters             tableNewIters
 	tableNewRangeKeyIter keyspanimpl.TableNewSpanIter
 
@@ -1408,12 +1408,10 @@ func (i *Iterator) constructPointIter(
 	// accumulator for this iterator's configured category and QoS. All SSTable
 	// iterators created by this Iterator will accumulate their stats to it as
 	// they Close during iteration.
-	if collector := i.fc.dbOpts.sstStatsCollector; collector != nil {
-		internalOpts.iterStatsAccumulator = collector.Accumulator(
-			uint64(uintptr(unsafe.Pointer(i))),
-			i.opts.Category,
-		)
-	}
+	internalOpts.iterStatsAccumulator = i.fc.SSTStatsCollector().Accumulator(
+		uint64(uintptr(unsafe.Pointer(i))),
+		i.opts.Category,
+	)
 	if i.opts.RangeKeyMasking.Filter != nil {
 		internalOpts.boundLimitedFilter = &i.rangeKeyMasking
 	}
@@ -1653,7 +1651,7 @@ func (d *DB) Close() error {
 	d.closed.Store(errors.WithStack(ErrClosed))
 	close(d.closedCh)
 
-	defer d.opts.Cache.Unref()
+	defer d.cacheHandle.Close()
 
 	for d.mu.compact.compactingCount > 0 || d.mu.compact.downloadingCount > 0 || d.mu.compact.flushing {
 		d.mu.compact.cond.Wait()
@@ -1670,7 +1668,6 @@ func (d *DB) Close() error {
 		err = errors.Errorf("pebble: %d unexpected in-progress compactions", errors.Safe(n))
 	}
 	err = firstError(err, d.mu.formatVers.marker.Close())
-	err = firstError(err, d.fileCache.close())
 	if !d.opts.ReadOnly {
 		if d.mu.log.writer != nil {
 			_, err2 := d.mu.log.writer.Close()
@@ -1760,6 +1757,8 @@ func (d *DB) Close() error {
 		err = firstError(err, errors.Errorf("leaked snapshots: %d open snapshots on DB %p", v, d))
 	}
 
+	err = firstError(err, d.fileCache.Close())
+
 	return err
 }
 
@@ -2047,9 +2046,9 @@ func (d *DB) Metrics() *Metrics {
 	d.mu.Unlock()
 
 	metrics.BlockCache = d.opts.Cache.Metrics()
-	metrics.FileCache, metrics.Filter = d.fileCache.metrics()
-	metrics.TableIters = int64(d.fileCache.iterCount())
-	metrics.CategoryStats = d.fileCache.dbOpts.sstStatsCollector.GetStats()
+	metrics.FileCache, metrics.Filter = d.fileCache.Metrics()
+	metrics.TableIters = d.fileCache.IterCount()
+	metrics.CategoryStats = d.fileCache.SSTStatsCollector().GetStats()
 
 	metrics.SecondaryCacheMetrics = d.objProvider.Metrics()
 
@@ -2214,7 +2213,7 @@ func (d *DB) SSTables(opts ...SSTablesOption) ([][]SSTableInfo, error) {
 			}
 			destTables[j].Virtual = m.Virtual
 			destTables[j].BackingSSTNum = m.FileBacking.DiskFileNum
-			objMeta, err := d.objProvider.Lookup(fileTypeTable, m.FileBacking.DiskFileNum)
+			objMeta, err := d.objProvider.Lookup(base.FileTypeTable, m.FileBacking.DiskFileNum)
 			if err != nil {
 				return nil, err
 			}
@@ -2905,7 +2904,7 @@ func (d *DB) checkVirtualBounds(m *fileMetadata) {
 		return
 	}
 
-	objMeta, err := d.objProvider.Lookup(fileTypeTable, m.FileBacking.DiskFileNum)
+	objMeta, err := d.objProvider.Lookup(base.FileTypeTable, m.FileBacking.DiskFileNum)
 	if err != nil {
 		panic(err)
 	}
diff --git a/event.go b/event.go
index bda74d33..e0213aad 100644
--- a/event.go
+++ b/event.go
@@ -20,6 +20,9 @@ import (
 	"github.com/cockroachdb/redact"
 )
 
+// FileNum is an identifier for a file within a database.
+type FileNum = base.FileNum
+
 // TableInfo exports the manifest.TableInfo type.
 type TableInfo = manifest.TableInfo
 
diff --git a/external_iterator.go b/external_iterator.go
index 4d136c85..b3cba3b0 100644
--- a/external_iterator.go
+++ b/external_iterator.go
@@ -48,15 +48,10 @@ func NewExternalIterWithContext(
 		}
 	}
 
+	ro := o.MakeReaderOptions()
 	var readers [][]*sstable.Reader
-
-	seqNumOffset := 0
-	for _, levelFiles := range files {
-		seqNumOffset += len(levelFiles)
-	}
 	for _, levelFiles := range files {
-		seqNumOffset -= len(levelFiles)
-		subReaders, err := openExternalTables(ctx, o, levelFiles, seqNumOffset, o.MakeReaderOptions())
+		subReaders, err := openExternalTables(ctx, levelFiles, ro)
 		readers = append(readers, subReaders)
 		if err != nil {
 			// Close all the opened readers.
@@ -81,9 +76,9 @@ func NewExternalIterWithContext(
 		prefixOrFullSeekKey: buf.prefixOrFullSeekKey,
 		boundsBuf:           buf.boundsBuf,
 		batch:               nil,
-		// Add the readers to the Iterator so that Close closes them, and
-		// SetOptions can re-construct iterators from them.
-		externalReaders: readers,
+		// Add the external iter state to the Iterator so that Close closes it,
+		// and SetOptions can re-construct iterators using its state.
+		externalIter: &externalIterState{readers: readers},
 		newIters: func(context.Context, *manifest.FileMetadata, *IterOptions,
 			internalIterOpts, iterKinds) (iterSet, error) {
 			// NB: External iterators are currently constructed without any
@@ -96,6 +91,8 @@ func NewExternalIterWithContext(
 		},
 		seqNum: base.SeqNumMax,
 	}
+	dbi.externalIter.bufferPool.Init(2)
+
 	if iterOpts != nil {
 		dbi.opts = *iterOpts
 		dbi.processBounds(iterOpts.LowerBound, iterOpts.UpperBound)
@@ -107,6 +104,25 @@ func NewExternalIterWithContext(
 	return dbi, nil
 }
 
+// externalIterState encapsulates state that is specific to external iterators.
+// An external *pebble.Iterator maintains a pointer to the externalIterState and
+// calls Close when the Iterator is Closed, providing an opportuntity for the
+// external iterator to release resources particular to external iterators.
+type externalIterState struct {
+	bufferPool block.BufferPool
+	readers    [][]*sstable.Reader
+}
+
+func (e *externalIterState) Close() (err error) {
+	for _, readers := range e.readers {
+		for _, r := range readers {
+			err = firstError(err, r.Close())
+		}
+	}
+	e.bufferPool.Release()
+	return err
+}
+
 func validateExternalIterOpts(iterOpts *IterOptions) error {
 	switch {
 	case iterOpts.PointKeyFilters != nil:
@@ -121,7 +137,9 @@ func validateExternalIterOpts(iterOpts *IterOptions) error {
 	return nil
 }
 
-func createExternalPointIter(ctx context.Context, it *Iterator) (topLevelIterator, error) {
+func createExternalPointIter(
+	ctx context.Context, it *Iterator, readEnv block.ReadEnv,
+) (topLevelIterator, error) {
 	// TODO(jackson): In some instances we could generate fewer levels by using
 	// L0Sublevels code to organize nonoverlapping files into the same level.
 	// This would allow us to use levelIters and keep a smaller set of data and
@@ -135,20 +153,15 @@ func createExternalPointIter(ctx context.Context, it *Iterator) (topLevelIterato
 	}
 	mlevels := it.alloc.mlevels[:0]
 
-	// TODO(jackson): External iterators never provide categorized iterator
-	// stats today because they exist outside the context of a *DB. If the
-	// sstables being read are on the physical filesystem, we may still want to
-	// thread a CategoryStatsCollector through so that we collect their stats.
-
-	if len(it.externalReaders) > cap(mlevels) {
-		mlevels = make([]mergingIterLevel, 0, len(it.externalReaders))
+	if len(it.externalIter.readers) > cap(mlevels) {
+		mlevels = make([]mergingIterLevel, 0, len(it.externalIter.readers))
 	}
 	// We set a synthetic sequence number, with lower levels having higer numbers.
 	seqNum := 0
-	for _, readers := range it.externalReaders {
+	for _, readers := range it.externalIter.readers {
 		seqNum += len(readers)
 	}
-	for _, readers := range it.externalReaders {
+	for _, readers := range it.externalIter.readers {
 		for _, r := range readers {
 			var (
 				rangeDelIter keyspan.FragmentIterator
@@ -164,15 +177,13 @@ func createExternalPointIter(ctx context.Context, it *Iterator) (topLevelIterato
 			seqNum--
 			pointIter, err = r.NewPointIter(
 				ctx, transforms, it.opts.LowerBound, it.opts.UpperBound, nil, /* BlockPropertiesFilterer */
-				sstable.NeverUseFilterBlock,
-				block.ReadEnv{Stats: &it.stats.InternalStats, IterStats: nil},
-				sstable.MakeTrivialReaderProvider(r))
+				sstable.NeverUseFilterBlock, readEnv, sstable.MakeTrivialReaderProvider(r))
 			if err != nil {
 				return nil, err
 			}
 			rangeDelIter, err = r.NewRawRangeDelIter(ctx, sstable.FragmentIterTransforms{
 				SyntheticSeqNum: sstable.SyntheticSeqNum(seqNum),
-			}, block.ReadEnv{Stats: &it.stats.InternalStats, IterStats: nil})
+			}, readEnv)
 			if err != nil {
 				return nil, err
 			}
@@ -192,7 +203,16 @@ func createExternalPointIter(ctx context.Context, it *Iterator) (topLevelIterato
 }
 
 func finishInitializingExternal(ctx context.Context, it *Iterator) error {
-	pointIter, err := createExternalPointIter(ctx, it)
+	readEnv := block.ReadEnv{
+		Stats: &it.stats.InternalStats,
+		// TODO(jackson): External iterators never provide categorized iterator
+		// stats today because they exist outside the context of a *DB. If the
+		// sstables being read are on the physical filesystem, we may still want to
+		// thread a CategoryStatsCollector through so that we collect their stats.
+		IterStats:  nil,
+		BufferPool: &it.externalIter.bufferPool,
+	}
+	pointIter, err := createExternalPointIter(ctx, it, readEnv)
 	if err != nil {
 		return err
 	}
@@ -214,14 +234,14 @@ func finishInitializingExternal(ctx context.Context, it *Iterator) error {
 			// this optimization.
 			// We set a synthetic sequence number, with lower levels having higer numbers.
 			seqNum := 0
-			for _, readers := range it.externalReaders {
+			for _, readers := range it.externalIter.readers {
 				seqNum += len(readers)
 			}
-			for _, readers := range it.externalReaders {
+			for _, readers := range it.externalIter.readers {
 				for _, r := range readers {
 					transforms := sstable.FragmentIterTransforms{SyntheticSeqNum: sstable.SyntheticSeqNum(seqNum)}
 					seqNum--
-					if rki, err := r.NewRawRangeKeyIter(ctx, transforms, block.ReadEnv{Stats: &it.stats.InternalStats, IterStats: nil}); err != nil {
+					if rki, err := r.NewRawRangeKeyIter(ctx, transforms, readEnv); err != nil {
 						return err
 					} else if rki != nil {
 						rangeKeyIters = append(rangeKeyIters, rki)
@@ -257,11 +277,7 @@ func finishInitializingExternal(ctx context.Context, it *Iterator) error {
 }
 
 func openExternalTables(
-	ctx context.Context,
-	o *Options,
-	files []sstable.ReadableFile,
-	seqNumOffset int,
-	readerOpts sstable.ReaderOptions,
+	ctx context.Context, files []sstable.ReadableFile, readerOpts sstable.ReaderOptions,
 ) (readers []*sstable.Reader, err error) {
 	readers = make([]*sstable.Reader, 0, len(files))
 	for i := range files {
diff --git a/file_cache.go b/file_cache.go
index e09bda2c..856cdf56 100644
--- a/file_cache.go
+++ b/file_cache.go
@@ -52,7 +52,7 @@ var emptyKeyspanIter = &errorKeyspanIter{err: nil}
 // On error, all iterators are nil.
 //
 // The only (non-test) implementation of tableNewIters is
-// fileCacheContainer.newIters().
+// fileCacheHandle.newIters().
 type tableNewIters func(
 	ctx context.Context,
 	file *manifest.FileMetadata,
@@ -85,116 +85,96 @@ func tableNewRangeKeyIter(newIters tableNewIters) keyspanimpl.TableNewSpanIter {
 	}
 }
 
-// fileCacheOpts contains the db specific fields of a file cache. This is stored
-// in the fileCacheContainer along with the file cache.
-//
-// NB: It is important to make sure that the fields in this struct are
-// read-only. Since the fields here are shared by every single fileCacheShard,
-// if non read-only fields are updated, we could have unnecessary evictions of
-// those fields, and the surrounding fields from the CPU caches.
-type fileCacheOpts struct {
-	// iterCount keeps track of how many iterators are open. It is used to keep
-	// track of leaked iterators on a per-db level.
-	iterCount *atomic.Int32
-
-	loggerAndTracer   LoggerAndTracer
-	cache             *cache.Cache
-	cacheID           cache.ID
-	objProvider       objstorage.Provider
-	readerOpts        sstable.ReaderOptions
-	sstStatsCollector *block.CategoryStatsCollector
-}
-
-// fileCacheContainer contains the file cache and fields which are unique to the
-// DB.
-type fileCacheContainer struct {
+// fileCacheHandle is used to access the file cache. Each DB has its own handle.
+type fileCacheHandle struct {
 	fileCache *FileCache
 
-	// dbOpts contains fields relevant to the file cache which are unique to
-	// each DB.
-	dbOpts fileCacheOpts
-}
-
-// newFileCacheContainer will panic if the underlying block cache in the file
-// cache doesn't match Options.Cache.
-func newFileCacheContainer(
-	fc *FileCache,
-	cacheID cache.ID,
-	objProvider objstorage.Provider,
-	opts *Options,
-	size int,
-	sstStatsCollector *block.CategoryStatsCollector,
-) *fileCacheContainer {
-	// We will release a ref to the file cache acquired here when
-	// fileCacheContainer.close is called.
-	if fc != nil {
-		if fc.cache != opts.Cache {
-			panic("pebble: underlying cache for the file cache and db are different")
-		}
-		fc.Ref()
-	} else {
-		// NewFileCache should create a ref to fc which the container should
-		// drop whenever it is closed.
-		fc = NewFileCache(opts.Cache, opts.Experimental.FileCacheShards, size)
-	}
-
-	t := &fileCacheContainer{}
-	t.fileCache = fc
-	t.dbOpts.loggerAndTracer = opts.LoggerAndTracer
-	t.dbOpts.cache = opts.Cache
-	t.dbOpts.cacheID = cacheID
-	t.dbOpts.objProvider = objProvider
-	t.dbOpts.readerOpts = opts.MakeReaderOptions()
-	t.dbOpts.readerOpts.FilterMetricsTracker = &sstable.FilterMetricsTracker{}
-	t.dbOpts.iterCount = new(atomic.Int32)
-	t.dbOpts.sstStatsCollector = sstStatsCollector
+	// The handle contains fields which are unique to each DB. Note that these get
+	// accessed from all shards, so keep read-only fields separate for read-write
+	// fields.
+	loggerAndTracer  LoggerAndTracer
+	blockCacheHandle *cache.Handle
+	objProvider      objstorage.Provider
+	readerOpts       sstable.ReaderOptions
+
+	// iterCount keeps track of how many iterators are open. It is used to keep
+	// track of leaked iterators on a per-db level.
+	iterCount         atomic.Int32
+	sstStatsCollector block.CategoryStatsCollector
+}
+
+// newHandle creates a handle for the FileCache which has its own options. Each
+// handle has its own set of files in the cache, separate from those of other
+// handles.
+// TODO(radu): don't pass entire options.
+func (c *FileCache) newHandle(
+	cacheHandle *cache.Handle, objProvider objstorage.Provider, opts *Options,
+) *fileCacheHandle {
+	c.Ref()
+
+	t := &fileCacheHandle{
+		fileCache:        c,
+		loggerAndTracer:  opts.LoggerAndTracer,
+		blockCacheHandle: cacheHandle,
+		objProvider:      objProvider,
+	}
+	t.readerOpts = opts.MakeReaderOptions()
+	t.readerOpts.FilterMetricsTracker = &sstable.FilterMetricsTracker{}
 	return t
 }
 
-// Before calling close, make sure that there will be no further need
+// Close the handle, make sure that there will be no further need
 // to access any of the files associated with the store.
-func (c *fileCacheContainer) close() error {
+func (h *fileCacheHandle) Close() error {
 	// We want to do some cleanup work here. Check for leaked iterators
 	// by the DB using this container. Note that we'll still perform cleanup
 	// below in the case that there are leaked iterators.
 	var err error
-	if v := c.dbOpts.iterCount.Load(); v > 0 {
+	if v := h.iterCount.Load(); v > 0 {
 		err = errors.Errorf("leaked iterators: %d", errors.Safe(v))
 	}
 
 	// Release nodes here.
-	for _, shard := range c.fileCache.shards {
+	for _, shard := range h.fileCache.shards {
 		if shard != nil {
-			shard.removeDB(&c.dbOpts)
+			shard.removeDB(h)
 		}
 	}
-	return firstError(err, c.fileCache.Unref())
+	err = firstError(err, h.fileCache.Unref())
+	// TODO(radu): we have to tolerate metrics() calls after close (see
+	// https://github.com/cockroachdb/cockroach/issues/140454).
+	// *h = fileCacheHandle{}
+	return err
 }
 
-func (c *fileCacheContainer) newIters(
+func (h *fileCacheHandle) newIters(
 	ctx context.Context,
 	file *manifest.FileMetadata,
 	opts *IterOptions,
 	internalOpts internalIterOpts,
 	kinds iterKinds,
 ) (iterSet, error) {
-	return c.fileCache.getShard(file.FileBacking.DiskFileNum).newIters(ctx, file, opts, internalOpts, &c.dbOpts, kinds)
+	return h.fileCache.getShard(file.FileBacking.DiskFileNum).newIters(ctx, file, opts, internalOpts, h, kinds)
 }
 
 // getTableProperties returns the properties associated with the backing physical
 // table if the input metadata belongs to a virtual sstable.
-func (c *fileCacheContainer) getTableProperties(file *fileMetadata) (*sstable.Properties, error) {
-	return c.fileCache.getShard(file.FileBacking.DiskFileNum).getTableProperties(file, &c.dbOpts)
+func (h *fileCacheHandle) getTableProperties(file *fileMetadata) (*sstable.Properties, error) {
+	return h.fileCache.getShard(file.FileBacking.DiskFileNum).getTableProperties(file, h)
+}
+
+func (h *fileCacheHandle) evict(fileNum base.DiskFileNum) {
+	h.fileCache.getShard(fileNum).evict(fileNum, h, false)
 }
 
-func (c *fileCacheContainer) evict(fileNum base.DiskFileNum) {
-	c.fileCache.getShard(fileNum).evict(fileNum, &c.dbOpts, false)
+func (h *fileCacheHandle) SSTStatsCollector() *block.CategoryStatsCollector {
+	return &h.sstStatsCollector
 }
 
-func (c *fileCacheContainer) metrics() (CacheMetrics, FilterMetrics) {
+func (h *fileCacheHandle) Metrics() (CacheMetrics, FilterMetrics) {
 	var m CacheMetrics
-	for i := range c.fileCache.shards {
-		s := c.fileCache.shards[i]
+	for i := range h.fileCache.shards {
+		s := h.fileCache.shards[i]
 		s.mu.RLock()
 		m.Count += int64(len(s.mu.nodes))
 		s.mu.RUnlock()
@@ -202,14 +182,14 @@ func (c *fileCacheContainer) metrics() (CacheMetrics, FilterMetrics) {
 		m.Misses += s.misses.Load()
 	}
 	m.Size = m.Count * int64(unsafe.Sizeof(fileCacheNode{})+unsafe.Sizeof(fileCacheValue{})+unsafe.Sizeof(sstable.Reader{}))
-	f := c.dbOpts.readerOpts.FilterMetricsTracker.Load()
+	f := h.readerOpts.FilterMetricsTracker.Load()
 	return m, f
 }
 
-func (c *fileCacheContainer) estimateSize(
+func (h *fileCacheHandle) estimateSize(
 	meta *fileMetadata, lower, upper []byte,
 ) (size uint64, err error) {
-	c.withCommonReader(meta, func(cr sstable.CommonReader) error {
+	h.withCommonReader(meta, func(cr sstable.CommonReader) error {
 		size, err = cr.EstimateDiskUsage(lower, upper)
 		return err
 	})
@@ -230,11 +210,11 @@ func createCommonReader(v *fileCacheValue, file *fileMetadata) sstable.CommonRea
 	return cr
 }
 
-func (c *fileCacheContainer) withCommonReader(
+func (h *fileCacheHandle) withCommonReader(
 	meta *fileMetadata, fn func(sstable.CommonReader) error,
 ) error {
-	s := c.fileCache.getShard(meta.FileBacking.DiskFileNum)
-	v := s.findNode(context.TODO(), meta.FileBacking, &c.dbOpts)
+	s := h.fileCache.getShard(meta.FileBacking.DiskFileNum)
+	v := s.findNode(context.TODO(), meta.FileBacking, h)
 	defer s.unrefValue(v)
 	if v.err != nil {
 		return v.err
@@ -242,9 +222,9 @@ func (c *fileCacheContainer) withCommonReader(
 	return fn(createCommonReader(v, meta))
 }
 
-func (c *fileCacheContainer) withReader(meta physicalMeta, fn func(*sstable.Reader) error) error {
-	s := c.fileCache.getShard(meta.FileBacking.DiskFileNum)
-	v := s.findNode(context.TODO(), meta.FileBacking, &c.dbOpts)
+func (h *fileCacheHandle) withReader(meta physicalMeta, fn func(*sstable.Reader) error) error {
+	s := h.fileCache.getShard(meta.FileBacking.DiskFileNum)
+	v := s.findNode(context.TODO(), meta.FileBacking, h)
 	defer s.unrefValue(v)
 	if v.err != nil {
 		return v.err
@@ -253,25 +233,24 @@ func (c *fileCacheContainer) withReader(meta physicalMeta, fn func(*sstable.Read
 }
 
 // withVirtualReader fetches a VirtualReader associated with a virtual sstable.
-func (c *fileCacheContainer) withVirtualReader(
+func (h *fileCacheHandle) withVirtualReader(
 	meta virtualMeta, fn func(sstable.VirtualReader) error,
 ) error {
-	s := c.fileCache.getShard(meta.FileBacking.DiskFileNum)
-	v := s.findNode(context.TODO(), meta.FileBacking, &c.dbOpts)
+	s := h.fileCache.getShard(meta.FileBacking.DiskFileNum)
+	v := s.findNode(context.TODO(), meta.FileBacking, h)
 	defer s.unrefValue(v)
 	if v.err != nil {
 		return v.err
 	}
-	provider := c.dbOpts.objProvider
-	objMeta, err := provider.Lookup(fileTypeTable, meta.FileBacking.DiskFileNum)
+	objMeta, err := h.objProvider.Lookup(base.FileTypeTable, meta.FileBacking.DiskFileNum)
 	if err != nil {
 		return err
 	}
 	return fn(sstable.MakeVirtualReader(v.mustSSTableReader(), meta.VirtualReaderParams(objMeta.IsShared())))
 }
 
-func (c *fileCacheContainer) iterCount() int64 {
-	return int64(c.dbOpts.iterCount.Load())
+func (h *fileCacheHandle) IterCount() int64 {
+	return int64(h.iterCount.Load())
 }
 
 // FileCache is a shareable cache for open files. Open files are exclusively
@@ -279,7 +258,6 @@ func (c *fileCacheContainer) iterCount() int64 {
 type FileCache struct {
 	refs atomic.Int64
 
-	cache  *Cache
 	shards []*fileCacheShard
 }
 
@@ -310,9 +288,6 @@ func (c *FileCache) Unref() error {
 			err = firstError(err, c.shards[i].Close())
 		}
 
-		// Unref the cache which we create a reference to when the file cache is
-		// first instantiated.
-		c.cache.Unref()
 		return err
 	}
 	return nil
@@ -321,7 +296,7 @@ func (c *FileCache) Unref() error {
 // NewFileCache will create a new file cache with one outstanding reference. It
 // is the callers responsibility to call Unref if they will no longer hold a
 // reference to the file cache.
-func NewFileCache(cache *Cache, numShards int, size int) *FileCache {
+func NewFileCache(numShards int, size int) *FileCache {
 	if size == 0 {
 		panic("pebble: cannot create a file cache of size 0")
 	} else if numShards == 0 {
@@ -329,8 +304,6 @@ func NewFileCache(cache *Cache, numShards int, size int) *FileCache {
 	}
 
 	c := &FileCache{}
-	c.cache = cache
-	c.cache.Ref()
 
 	c.shards = make([]*fileCacheShard, numShards)
 	for i := range c.shards {
@@ -349,7 +322,7 @@ func (c *FileCache) getShard(fileNum base.DiskFileNum) *fileCacheShard {
 }
 
 type fileCacheKey struct {
-	cacheID cache.ID
+	handle  *fileCacheHandle
 	fileNum base.DiskFileNum
 }
 
@@ -363,8 +336,6 @@ type fileCacheShard struct {
 	mu struct {
 		sync.RWMutex
 		nodes map[fileCacheKey]*fileCacheNode
-		// The iters map is only created and populated in race builds.
-		iters map[io.Closer][]byte
 
 		handHot  *fileCacheNode
 		handCold *fileCacheNode
@@ -374,6 +345,11 @@ type fileCacheShard struct {
 		sizeHot    int
 		sizeCold   int
 		sizeTest   int
+
+		race struct {
+			// The iters map is only created and populated in race builds.
+			iters map[any][]byte
+		}
 	}
 	releasing       sync.WaitGroup
 	releasingCh     chan *fileCacheValue
@@ -390,7 +366,7 @@ func (c *fileCacheShard) init(size int) {
 	go c.releaseLoop()
 
 	if invariants.RaceEnabled {
-		c.mu.iters = make(map[io.Closer][]byte)
+		c.mu.race.iters = make(map[any][]byte)
 	}
 }
 
@@ -431,7 +407,7 @@ func (c *fileCacheShard) newIters(
 	file *manifest.FileMetadata,
 	opts *IterOptions,
 	internalOpts internalIterOpts,
-	dbOpts *fileCacheOpts,
+	handle *fileCacheHandle,
 	kinds iterKinds,
 ) (iterSet, error) {
 	// TODO(sumeer): constructing the Reader should also use a plumbed context,
@@ -442,7 +418,7 @@ func (c *fileCacheShard) newIters(
 	// refCount. If opening the underlying table resulted in error, then we
 	// decrement this straight away. Otherwise, we pass that responsibility to
 	// the sstable iterator, which decrements when it is closed.
-	v := c.findNode(ctx, file.FileBacking, dbOpts)
+	v := c.findNode(ctx, file.FileBacking, handle)
 	if v.err != nil {
 		defer c.unrefValue(v)
 		return iterSet{}, v.err
@@ -457,10 +433,10 @@ func (c *fileCacheShard) newIters(
 		iters.rangeKey, err = newRangeKeyIter(ctx, r, file, cr, opts.SpanIterOptions(), internalOpts)
 	}
 	if kinds.RangeDeletion() && file.HasPointKeys && err == nil {
-		iters.rangeDeletion, err = newRangeDelIter(ctx, file, cr, dbOpts, internalOpts)
+		iters.rangeDeletion, err = newRangeDelIter(ctx, file, cr, handle, internalOpts)
 	}
 	if kinds.Point() && err == nil {
-		iters.point, err = c.newPointIter(ctx, v, file, cr, opts, internalOpts, dbOpts)
+		iters.point, err = c.newPointIter(ctx, v, file, cr, opts, internalOpts, handle)
 	}
 	if err != nil {
 		// NB: There's a subtlety here: Because the point iterator is the last
@@ -494,7 +470,7 @@ func (c *fileCacheShard) newPointIter(
 	cr sstable.CommonReader,
 	opts *IterOptions,
 	internalOpts internalIterOpts,
-	dbOpts *fileCacheOpts,
+	handle *fileCacheHandle,
 ) (internalIterator, error) {
 	var (
 		hideObsoletePoints bool = false
@@ -569,9 +545,8 @@ func (c *fileCacheShard) newPointIter(
 	transforms := file.IterTransforms()
 	transforms.HideObsoletePoints = hideObsoletePoints
 	iterStatsAccum := internalOpts.iterStatsAccumulator
-	if iterStatsAccum == nil && opts != nil && dbOpts.sstStatsCollector != nil {
-		iterStatsAccum = dbOpts.sstStatsCollector.Accumulator(
-			uint64(uintptr(unsafe.Pointer(r))), opts.Category)
+	if iterStatsAccum == nil && opts != nil {
+		iterStatsAccum = handle.SSTStatsCollector().Accumulator(uint64(uintptr(unsafe.Pointer(r))), opts.Category)
 	}
 	if internalOpts.compaction {
 		iter, err = cr.NewCompactionIter(transforms, block.ReadEnv{IterStats: iterStatsAccum, BufferPool: internalOpts.bufferPool}, &v.readerProvider)
@@ -587,10 +562,10 @@ func (c *fileCacheShard) newPointIter(
 	// care to avoid introducing an allocation here by adding a closure.
 	iter.SetCloseHook(v.closeHook)
 	c.iterCount.Add(1)
-	dbOpts.iterCount.Add(1)
+	handle.iterCount.Add(1)
 	if invariants.RaceEnabled {
 		c.mu.Lock()
-		c.mu.iters[iter] = debug.Stack()
+		c.mu.race.iters[iter] = debug.Stack()
 		c.mu.Unlock()
 	}
 	return iter, nil
@@ -603,7 +578,7 @@ func newRangeDelIter(
 	ctx context.Context,
 	file *manifest.FileMetadata,
 	cr sstable.CommonReader,
-	dbOpts *fileCacheOpts,
+	handle *fileCacheHandle,
 	internalOpts internalIterOpts,
 ) (keyspan.FragmentIterator, error) {
 	// NB: range-del iterator does not maintain a reference to the table, nor
@@ -620,8 +595,8 @@ func newRangeDelIter(
 	// Assert expected bounds in tests.
 	if invariants.Sometimes(50) && rangeDelIter != nil {
 		cmp := base.DefaultComparer.Compare
-		if dbOpts.readerOpts.Comparer != nil {
-			cmp = dbOpts.readerOpts.Comparer.Compare
+		if handle.readerOpts.Comparer != nil {
+			cmp = handle.readerOpts.Comparer.Compare
 		}
 		rangeDelIter = keyspan.AssertBounds(
 			rangeDelIter, file.SmallestPointKey, file.LargestPointKey.UserKey, cmp,
@@ -668,7 +643,7 @@ func newRangeKeyIter(
 // specific table.
 type tableCacheShardReaderProvider struct {
 	c              *fileCacheShard
-	dbOpts         *fileCacheOpts
+	handle         *fileCacheHandle
 	backingFileNum base.DiskFileNum
 
 	mu struct {
@@ -685,10 +660,10 @@ type tableCacheShardReaderProvider struct {
 var _ valblk.ReaderProvider = &tableCacheShardReaderProvider{}
 
 func (rp *tableCacheShardReaderProvider) init(
-	c *fileCacheShard, dbOpts *fileCacheOpts, backingFileNum base.DiskFileNum,
+	c *fileCacheShard, handle *fileCacheHandle, backingFileNum base.DiskFileNum,
 ) {
 	rp.c = c
-	rp.dbOpts = dbOpts
+	rp.handle = handle
 	rp.backingFileNum = backingFileNum
 	rp.mu.v = nil
 	rp.mu.refCount = 0
@@ -724,7 +699,7 @@ func (rp *tableCacheShardReaderProvider) GetReader(
 	// findNodeInternal will need to do IO to initialize a new Reader. We hold
 	// rp.mu during this time so that concurrent GetReader calls block until the
 	// Reader is created.
-	v := rp.c.findNodeInternal(ctx, rp.backingFileNum, rp.dbOpts)
+	v := rp.c.findNodeInternal(ctx, rp.backingFileNum, rp.handle)
 	if v.err != nil {
 		defer rp.c.unrefValue(v)
 		return nil, v.err
@@ -750,10 +725,10 @@ func (rp *tableCacheShardReaderProvider) Close() {
 
 // getTableProperties return sst table properties for target file.
 func (c *fileCacheShard) getTableProperties(
-	file *fileMetadata, dbOpts *fileCacheOpts,
+	file *fileMetadata, handle *fileCacheHandle,
 ) (*sstable.Properties, error) {
 	// Calling findNode gives us the responsibility of decrementing v's refCount here
-	v := c.findNode(context.TODO(), file.FileBacking, dbOpts)
+	v := c.findNode(context.TODO(), file.FileBacking, handle)
 	defer c.unrefValue(v)
 
 	if v.err != nil {
@@ -776,7 +751,7 @@ func (c *fileCacheShard) releaseNode(n *fileCacheNode) {
 //
 // c.mu must be held when calling this.
 func (c *fileCacheShard) unlinkNode(n *fileCacheNode) {
-	key := fileCacheKey{n.cacheID, n.fileNum}
+	key := fileCacheKey{n.handle, n.fileNum}
 	delete(c.mu.nodes, key)
 
 	switch n.ptype {
@@ -831,7 +806,7 @@ func (c *fileCacheShard) unrefValue(v *fileCacheValue) {
 // that node if it didn't already exist. The caller is responsible for
 // decrementing the returned node's refCount.
 func (c *fileCacheShard) findNode(
-	ctx context.Context, b *fileBacking, dbOpts *fileCacheOpts,
+	ctx context.Context, b *fileBacking, handle *fileCacheHandle,
 ) *fileCacheValue {
 	// The backing must have a positive refcount (otherwise it could be deleted
 	// at any time).
@@ -843,15 +818,15 @@ func (c *fileCacheShard) findNode(
 	// be passed to findNodeInternal.
 	backingFileNum := b.DiskFileNum
 
-	return c.findNodeInternal(ctx, backingFileNum, dbOpts)
+	return c.findNodeInternal(ctx, backingFileNum, handle)
 }
 
 func (c *fileCacheShard) findNodeInternal(
-	ctx context.Context, backingFileNum base.DiskFileNum, dbOpts *fileCacheOpts,
+	ctx context.Context, backingFileNum base.DiskFileNum, handle *fileCacheHandle,
 ) *fileCacheValue {
 	// Fast-path for a hit in the cache.
 	c.mu.RLock()
-	key := fileCacheKey{dbOpts.cacheID, backingFileNum}
+	key := fileCacheKey{handle, backingFileNum}
 	if n := c.mu.nodes[key]; n != nil && n.value != nil {
 		// Fast-path hit.
 		//
@@ -875,8 +850,9 @@ func (c *fileCacheShard) findNodeInternal(
 		n = &fileCacheNode{
 			fileNum: backingFileNum,
 			ptype:   fileCacheNodeCold,
+			handle:  handle,
 		}
-		c.addNode(n, dbOpts)
+		c.addNode(n)
 		c.mu.sizeCold++
 
 	case n.value != nil:
@@ -901,7 +877,8 @@ func (c *fileCacheShard) findNodeInternal(
 
 		n.referenced.Store(false)
 		n.ptype = fileCacheNodeHot
-		c.addNode(n, dbOpts)
+		n.handle = handle
+		c.addNode(n)
 		c.mu.sizeHot++
 	}
 
@@ -910,20 +887,19 @@ func (c *fileCacheShard) findNodeInternal(
 	v := &fileCacheValue{
 		loaded: make(chan struct{}),
 	}
-	v.readerProvider.init(c, dbOpts, backingFileNum)
+	v.readerProvider.init(c, handle, backingFileNum)
 	v.refCount.Store(2)
 	// Cache the closure invoked when an iterator is closed. This avoids an
 	// allocation on every call to newIters.
-	v.closeHook = func(i sstable.Iterator) error {
+	v.closeHook = func(i any) {
 		if invariants.RaceEnabled {
 			c.mu.Lock()
-			delete(c.mu.iters, i)
+			delete(c.mu.race.iters, i)
 			c.mu.Unlock()
 		}
 		c.unrefValue(v)
 		c.iterCount.Add(-1)
-		dbOpts.iterCount.Add(-1)
-		return nil
+		handle.iterCount.Add(-1)
 	}
 	n.value = v
 
@@ -931,14 +907,13 @@ func (c *fileCacheShard) findNodeInternal(
 
 	// Note adding to the cache lists must complete before we begin loading the
 	// table as a failure during load will result in the node being unlinked.
-	v.load(ctx, backingFileNum, c, dbOpts)
+	v.load(ctx, backingFileNum, c, handle)
 	return v
 }
 
-func (c *fileCacheShard) addNode(n *fileCacheNode, dbOpts *fileCacheOpts) {
+func (c *fileCacheShard) addNode(n *fileCacheNode) {
 	c.evictNodes()
-	n.cacheID = dbOpts.cacheID
-	key := fileCacheKey{n.cacheID, n.fileNum}
+	key := fileCacheKey{n.handle, n.fileNum}
 	c.mu.nodes[key] = n
 
 	n.links.next = n
@@ -1032,9 +1007,9 @@ func (c *fileCacheShard) runHandTest() {
 	c.mu.handTest = c.mu.handTest.next()
 }
 
-func (c *fileCacheShard) evict(fileNum base.DiskFileNum, dbOpts *fileCacheOpts, allowLeak bool) {
+func (c *fileCacheShard) evict(fileNum base.DiskFileNum, handle *fileCacheHandle, allowLeak bool) {
 	c.mu.Lock()
-	key := fileCacheKey{dbOpts.cacheID, fileNum}
+	key := fileCacheKey{handle, fileNum}
 	n := c.mu.nodes[key]
 	var v *fileCacheValue
 	if n != nil {
@@ -1048,7 +1023,7 @@ func (c *fileCacheShard) evict(fileNum base.DiskFileNum, dbOpts *fileCacheOpts,
 		if v != nil {
 			if !allowLeak {
 				if t := v.refCount.Add(-1); t != 0 {
-					dbOpts.loggerAndTracer.Fatalf("sstable %s: refcount is not zero: %d\n%s", fileNum, t, debug.Stack())
+					handle.loggerAndTracer.Fatalf("sstable %s: refcount is not zero: %d\n%s", fileNum, t, debug.Stack())
 				}
 			}
 			c.releasing.Add(1)
@@ -1061,13 +1036,12 @@ func (c *fileCacheShard) evict(fileNum base.DiskFileNum, dbOpts *fileCacheOpts,
 		v.release(c)
 	}
 
-	dbOpts.cache.EvictFile(dbOpts.cacheID, fileNum)
+	handle.blockCacheHandle.EvictFile(fileNum)
 }
 
-// removeDB evicts any nodes which have a reference to the DB
-// associated with dbOpts.cacheID. Make sure that there will
-// be no more accesses to the files associated with the DB.
-func (c *fileCacheShard) removeDB(dbOpts *fileCacheOpts) {
+// removeDB evicts any nodes associated with handle. Make sure that there will
+// be no more accesses to files associated with the handle.
+func (c *fileCacheShard) removeDB(handle *fileCacheHandle) {
 	var fileNums []base.DiskFileNum
 
 	c.mu.RLock()
@@ -1079,7 +1053,7 @@ func (c *fileCacheShard) removeDB(dbOpts *fileCacheOpts) {
 			firstNode = node
 		}
 
-		if node.cacheID == dbOpts.cacheID {
+		if node.handle == handle {
 			fileNums = append(fileNums, node.fileNum)
 		}
 		node = node.next()
@@ -1090,7 +1064,7 @@ func (c *fileCacheShard) removeDB(dbOpts *fileCacheOpts) {
 	// This should synchronously close all the files
 	// associated with the DB.
 	for _, fileNum := range fileNums {
-		c.evict(fileNum, dbOpts, true)
+		c.evict(fileNum, handle, true)
 	}
 }
 
@@ -1106,7 +1080,7 @@ func (c *fileCacheShard) Close() error {
 			err = errors.Errorf("leaked iterators: %d", errors.Safe(v))
 		} else {
 			var buf bytes.Buffer
-			for _, stack := range c.mu.iters {
+			for _, stack := range c.mu.race.iters {
 				fmt.Fprintf(&buf, "%s\n", stack)
 			}
 			err = errors.Errorf("leaked iterators: %d\n%s", errors.Safe(v), buf.String())
@@ -1146,7 +1120,7 @@ func (c *fileCacheShard) Close() error {
 }
 
 type fileCacheValue struct {
-	closeHook func(i sstable.Iterator) error
+	closeHook func(i any)
 	reader    io.Closer // *sstable.Reader
 	err       error
 	loaded    chan struct{}
@@ -1170,28 +1144,27 @@ func (v *fileCacheValue) mustSSTableReader() *sstable.Reader {
 }
 
 func (v *fileCacheValue) load(
-	ctx context.Context, backingFileNum base.DiskFileNum, c *fileCacheShard, dbOpts *fileCacheOpts,
+	ctx context.Context, backingFileNum base.DiskFileNum, c *fileCacheShard, handle *fileCacheHandle,
 ) {
 	// Try opening the file first.
 	var f objstorage.Readable
 	var r *sstable.Reader
 	var err error
-	f, err = dbOpts.objProvider.OpenForReading(
-		ctx, fileTypeTable, backingFileNum, objstorage.OpenOptions{MustExist: true},
+	f, err = handle.objProvider.OpenForReading(
+		ctx, base.FileTypeTable, backingFileNum, objstorage.OpenOptions{MustExist: true},
 	)
 	if err == nil {
-		o := dbOpts.readerOpts
+		o := handle.readerOpts
 		o.CacheOpts = sstableinternal.CacheOptions{
-			Cache:   dbOpts.cache,
-			CacheID: dbOpts.cacheID,
-			FileNum: backingFileNum,
+			CacheHandle: handle.blockCacheHandle,
+			FileNum:     backingFileNum,
 		}
 		r, err = sstable.NewReader(ctx, f, o)
 	}
 	if err == nil {
 		v.reader = r
 		var objMeta objstorage.ObjectMetadata
-		objMeta, err = dbOpts.objProvider.Lookup(fileTypeTable, backingFileNum)
+		objMeta, err = handle.objProvider.Lookup(base.FileTypeTable, backingFileNum)
 		v.isShared = objMeta.IsShared()
 	}
 	if err != nil {
@@ -1203,7 +1176,7 @@ func (v *fileCacheValue) load(
 		defer c.mu.Unlock()
 		// Lookup the node in the cache again as it might have already been
 		// removed.
-		key := fileCacheKey{dbOpts.cacheID, backingFileNum}
+		key := fileCacheKey{handle, backingFileNum}
 		n := c.mu.nodes[key]
 		if n != nil && n.value == v {
 			c.releaseNode(n)
@@ -1255,9 +1228,8 @@ type fileCacheNode struct {
 	// since the last time one of the clock hands swept it.
 	referenced atomic.Bool
 
-	// Storing the cache id associated with the DB instance here
-	// avoids the need to thread the dbOpts struct through many functions.
-	cacheID cache.ID
+	// A fileCacheNode is associated with a specific fileCacheHandle.
+	handle *fileCacheHandle
 }
 
 func (n *fileCacheNode) next() *fileCacheNode {
diff --git a/filenames.go b/filenames.go
deleted file mode 100644
index 86e7ed1d..00000000
--- a/filenames.go
+++ /dev/null
@@ -1,22 +0,0 @@
-// Copyright 2012 The LevelDB-Go and Pebble Authors. All rights reserved. Use
-// of this source code is governed by a BSD-style license that can be found in
-// the LICENSE file.
-
-package pebble
-
-import "github.com/cockroachdb/pebble/internal/base"
-
-type fileType = base.FileType
-
-// FileNum is an identifier for a file within a database.
-type FileNum = base.FileNum
-
-const (
-	fileTypeLog      = base.FileTypeLog
-	fileTypeLock     = base.FileTypeLock
-	fileTypeTable    = base.FileTypeTable
-	fileTypeManifest = base.FileTypeManifest
-	fileTypeOptions  = base.FileTypeOptions
-	fileTypeTemp     = base.FileTypeTemp
-	fileTypeOldTemp  = base.FileTypeOldTemp
-)
diff --git a/ingest.go b/ingest.go
index 08491994..ac4efb9d 100644
--- a/ingest.go
+++ b/ingest.go
@@ -98,6 +98,10 @@ func ingestValidateKey(opts *Options, key *InternalKey) error {
 		return base.CorruptionErrorf("pebble: external sstable has non-zero seqnum: %s",
 			key.Pretty(opts.Comparer.FormatKey))
 	}
+	if err := opts.Comparer.ValidateKey.Validate(key.UserKey); err != nil {
+		return base.CorruptionErrorf("pebble: external sstable has corrupted key: %s, %w",
+			key.Pretty(opts.Comparer.FormatKey), err)
+	}
 	return nil
 }
 
@@ -300,15 +304,14 @@ func ingestLoad1(
 	opts *Options,
 	fmv FormatMajorVersion,
 	readable objstorage.Readable,
-	cacheID cache.ID,
+	cacheHandle *cache.Handle,
 	fileNum base.FileNum,
 	rangeKeyValidator rangeKeyIngestValidator,
 ) (meta *fileMetadata, lastRangeKey keyspan.Span, err error) {
 	o := opts.MakeReaderOptions()
 	o.CacheOpts = sstableinternal.CacheOptions{
-		Cache:   opts.Cache,
-		CacheID: cacheID,
-		FileNum: base.PhysicalTableDiskFileNum(fileNum),
+		CacheHandle: cacheHandle,
+		FileNum:     base.PhysicalTableDiskFileNum(fileNum),
 	}
 	r, err := sstable.NewReader(ctx, readable, o)
 	if err != nil {
@@ -504,7 +507,7 @@ func ingestLoad(
 	paths []string,
 	shared []SharedSSTMeta,
 	external []ExternalFile,
-	cacheID cache.ID,
+	cacheHandle *cache.Handle,
 	pending []base.FileNum,
 ) (ingestLoadResult, error) {
 	localFileNums := pending[:len(paths)]
@@ -537,7 +540,7 @@ func ingestLoad(
 		if !shouldDisableRangeKeyChecks {
 			rangeKeyValidator = validateSuffixedBoundaries(opts.Comparer, lastRangeKey)
 		}
-		m, lastRangeKey, err = ingestLoad1(ctx, opts, fmv, readable, cacheID, localFileNums[i], rangeKeyValidator)
+		m, lastRangeKey, err = ingestLoad1(ctx, opts, fmv, readable, cacheHandle, localFileNums[i], rangeKeyValidator)
 		if err != nil {
 			return ingestLoadResult{}, err
 		}
@@ -675,7 +678,7 @@ func ingestSortAndVerify(cmp Compare, lr ingestLoadResult, exciseSpan KeyRange)
 func ingestCleanup(objProvider objstorage.Provider, meta []ingestLocalMeta) error {
 	var firstErr error
 	for i := range meta {
-		if err := objProvider.Remove(fileTypeTable, meta[i].FileBacking.DiskFileNum); err != nil {
+		if err := objProvider.Remove(base.FileTypeTable, meta[i].FileBacking.DiskFileNum); err != nil {
 			firstErr = firstError(firstErr, err)
 		}
 	}
@@ -693,7 +696,7 @@ func ingestLinkLocal(
 ) error {
 	for i := range localMetas {
 		objMeta, err := objProvider.LinkOrCopyFromLocal(
-			ctx, opts.FS, localMetas[i].path, fileTypeTable, localMetas[i].FileBacking.DiskFileNum,
+			ctx, opts.FS, localMetas[i].path, base.FileTypeTable, localMetas[i].FileBacking.DiskFileNum,
 			objstorage.CreateOptions{PreferSharedStorage: true},
 		)
 		if err != nil {
@@ -730,7 +733,7 @@ func (d *DB) ingestAttachRemote(jobID JobID, lr ingestLoadResult) error {
 		}
 		remoteObjs = append(remoteObjs, objstorage.RemoteObjectToAttach{
 			FileNum:  lr.shared[i].FileBacking.DiskFileNum,
-			FileType: fileTypeTable,
+			FileType: base.FileTypeTable,
 			Backing:  backing,
 		})
 	}
@@ -768,7 +771,7 @@ func (d *DB) ingestAttachRemote(jobID JobID, lr ingestLoadResult) error {
 
 		remoteObjs = append(remoteObjs, objstorage.RemoteObjectToAttach{
 			FileNum:  meta.FileBacking.DiskFileNum,
-			FileType: fileTypeTable,
+			FileType: base.FileTypeTable,
 			Backing:  providerBacking,
 		})
 	}
@@ -1459,7 +1462,7 @@ func (d *DB) ingest(
 
 	// Load the metadata for all the files being ingested. This step detects
 	// and elides empty sstables.
-	loadResult, err := ingestLoad(ctx, d.opts, d.FormatMajorVersion(), paths, shared, external, d.cacheID, pendingOutputs)
+	loadResult, err := ingestLoad(ctx, d.opts, d.FormatMajorVersion(), paths, shared, external, d.cacheHandle, pendingOutputs)
 	if err != nil {
 		return IngestOperationStats{}, err
 	}
@@ -1499,7 +1502,7 @@ func (d *DB) ingest(
 	// metaFlushableOverlaps is a map indicating which of the ingested sstables
 	// overlap some table in the flushable queue. It's used to approximate
 	// ingest-into-L0 stats when using flushable ingests.
-	metaFlushableOverlaps := make(map[FileNum]bool, loadResult.fileCount())
+	metaFlushableOverlaps := make(map[base.FileNum]bool, loadResult.fileCount())
 	var mem *flushableEntry
 	var mut *memTable
 	// asFlushable indicates whether the sstable was ingested as a flushable.
diff --git a/internal/base/cleaner.go b/internal/base/cleaner.go
index 1582ac80..e03b4e65 100644
--- a/internal/base/cleaner.go
+++ b/internal/base/cleaner.go
@@ -40,7 +40,7 @@ var _ NeedsFileContents = ArchiveCleaner{}
 // also write to the secondary. We should consider archiving to the primary.
 func (ArchiveCleaner) Clean(fs vfs.FS, fileType FileType, path string) error {
 	switch fileType {
-	case FileTypeLog, FileTypeManifest, FileTypeTable:
+	case FileTypeLog, FileTypeManifest, FileTypeTable, FileTypeBlob:
 		destDir := fs.PathJoin(fs.PathDir(path), "archive")
 
 		if err := fs.MkdirAll(destDir, 0755); err != nil {
diff --git a/internal/base/comparer.go b/internal/base/comparer.go
index 8e277e60..92713bad 100644
--- a/internal/base/comparer.go
+++ b/internal/base/comparer.go
@@ -12,6 +12,7 @@ import (
 	"strconv"
 	"unicode/utf8"
 
+	"github.com/cockroachdb/crlib/crbytes"
 	"github.com/cockroachdb/errors"
 )
 
@@ -124,11 +125,18 @@ type FormatValue func(key, value []byte) fmt.Formatter
 //
 // For example, if a and b are the []byte equivalents of the strings "black" and
 // "blue", then the function may append "blb" to dst.
+//
+// Callers must guarantee that len(a) > 0 and len(b) > 0.
 type Separator func(dst, a, b []byte) []byte
 
 // Successor appends to dst a shortened key k given a key a such that
 // Compare(a, k) <= 0. A simple implementation may return a unchanged.
 // The appended key k must be valid to pass to Compare.
+//
+// The parameter a may be an empty slice even if an empty slice was never
+// committed to Pebble and is not a valid key representation. If a is the empty
+// slice, Successor must return a valid key but otherwise has no other
+// constraints.
 type Successor func(dst, a []byte) []byte
 
 // ImmediateSuccessor is invoked with a prefix key ([Split(a) == len(a)]) and
@@ -229,6 +237,10 @@ type Comparer struct {
 	// FormatValue is optional.
 	FormatValue FormatValue
 
+	// ValidateKey is an optional function that determines whether a key is
+	// valid according to this Comparer's key encoding.
+	ValidateKey ValidateKey
+
 	// Name is the name of the comparer.
 	//
 	// The on-disk format stores the comparer name, and opening a database with a
@@ -308,14 +320,15 @@ var DefaultComparer = &Comparer{
 	FormatKey: DefaultFormatter,
 
 	Separator: func(dst, a, b []byte) []byte {
-		i, n := SharedPrefixLen(a, b), len(dst)
+		if len(a) == 0 || len(b) == 0 {
+			panic(errors.AssertionFailedf("empty keys"))
+		}
+
+		i := crbytes.CommonPrefix(a, b)
+		n := len(dst)
 		dst = append(dst, a...)
 
-		min := len(a)
-		if min > len(b) {
-			min = len(b)
-		}
-		if i >= min {
+		if i == len(a) || i == len(b) {
 			// Do not shorten if one string is a prefix of the other.
 			return dst
 		}
@@ -362,25 +375,6 @@ var DefaultComparer = &Comparer{
 	Name: "leveldb.BytewiseComparator",
 }
 
-// SharedPrefixLen returns the largest i such that a[:i] equals b[:i].
-// This function can be useful in implementing the Comparer interface.
-func SharedPrefixLen(a, b []byte) int {
-	i, n := 0, len(a)
-	if n > len(b) {
-		n = len(b)
-	}
-	asUint64 := func(c []byte, i int) uint64 {
-		return binary.LittleEndian.Uint64(c[i:])
-	}
-	for i < n-7 && asUint64(a, i) == asUint64(b, i) {
-		i += 8
-	}
-	for i < n && a[i] == b[i] {
-		i++
-	}
-	return i
-}
-
 // MinUserKey returns the smaller of two user keys. If one of the keys is nil,
 // the other one is returned.
 func MinUserKey(cmp Compare, a, b []byte) []byte {
@@ -439,13 +433,54 @@ func MakeAssertComparer(c Comparer) Comparer {
 		ComparePointSuffixes: c.ComparePointSuffixes,
 		CompareRangeSuffixes: c.CompareRangeSuffixes,
 		AbbreviatedKey:       c.AbbreviatedKey,
-		Separator:            c.Separator,
-		Successor:            c.Successor,
-		ImmediateSuccessor:   c.ImmediateSuccessor,
-		FormatKey:            c.FormatKey,
-		Split:                c.Split,
-		FormatValue:          c.FormatValue,
-		Name:                 c.Name,
+		Separator: func(dst, a, b []byte) []byte {
+			if len(a) == 0 || len(b) == 0 {
+				panic(errors.AssertionFailedf("empty keys"))
+			}
+			ret := c.Separator(dst, a, b)
+			// The Separator func must return a valid key.
+			c.ValidateKey.MustValidate(ret)
+			return ret
+		},
+		Successor: func(dst, a []byte) []byte {
+			ret := c.Successor(dst, a)
+			// The Successor func must return a valid key.
+			c.ValidateKey.MustValidate(ret)
+			return ret
+		},
+		ImmediateSuccessor: func(dst, a []byte) []byte {
+			ret := c.ImmediateSuccessor(dst, a)
+			// The ImmediateSuccessor func must return a valid key.
+			c.ValidateKey.MustValidate(ret)
+			return ret
+		},
+		FormatKey:   c.FormatKey,
+		Split:       c.Split,
+		FormatValue: c.FormatValue,
+		ValidateKey: c.ValidateKey,
+		Name:        c.Name,
+	}
+}
+
+// ValidateKey is a func that determines whether a key is valid according to a
+// particular key encoding. Returns nil if the provided key is a valid, full
+// user key. Implementations must be careful to not mutate the provided key.
+type ValidateKey func([]byte) error
+
+// Validate validates the provided user key. If the func is nil, Validate
+// returns nil.
+func (v ValidateKey) Validate(key []byte) error {
+	if v == nil {
+		return nil
+	}
+	return v(key)
+}
+
+// MustValidate validates the provided user key, panicking if the key is
+// invalid.
+func (v ValidateKey) MustValidate(key []byte) {
+	if err := v.Validate(key); err != nil {
+		panic(err)
 	}
 }
 
@@ -489,6 +524,13 @@ func CheckComparer(c *Comparer, prefixes [][]byte, suffixes [][]byte) error {
 		for i := 1; i < len(suffixes); i++ {
 			a := slices.Concat(p, suffixes[i-1])
 			b := slices.Concat(p, suffixes[i])
+			if err := c.ValidateKey.Validate(a); err != nil {
+				return err
+			}
+			if err := c.ValidateKey.Validate(b); err != nil {
+				return err
+			}
+
 			// Make sure the Compare function agrees with ComparePointSuffixes.
 			if cmp := c.Compare(a, b); cmp > 0 {
 				return errors.Errorf("Compare(%s, %s)=%d, expected <= 0", c.FormatKey(a), c.FormatKey(b), cmp)
@@ -500,9 +542,15 @@ func CheckComparer(c *Comparer, prefixes [][]byte, suffixes [][]byte) error {
 	for _, ap := range prefixes {
 		for _, as := range suffixes {
 			a := slices.Concat(ap, as)
+			if err := c.ValidateKey.Validate(a); err != nil {
+				return err
+			}
 			for _, bp := range prefixes {
 				for _, bs := range suffixes {
 					b := slices.Concat(bp, bs)
+					if err := c.ValidateKey.Validate(b); err != nil {
+						return err
+					}
 					result := c.Compare(a, b)
 					if (result == 0) != c.Equal(a, b) {
 						return errors.Errorf("Equal(%s, %s) doesn't agree with Compare", c.FormatKey(a), c.FormatKey(b))
diff --git a/internal/base/filenames.go b/internal/base/filenames.go
index 3b75d6c5..01051cc7 100644
--- a/internal/base/filenames.go
+++ b/internal/base/filenames.go
@@ -61,8 +61,44 @@ const (
 	FileTypeOptions
 	FileTypeOldTemp
 	FileTypeTemp
+	FileTypeBlob
 )
 
+var fileTypeStrings = [...]string{
+	FileTypeLog:      "log",
+	FileTypeLock:     "lock",
+	FileTypeTable:    "sstable",
+	FileTypeManifest: "manifest",
+	FileTypeOptions:  "options",
+	FileTypeOldTemp:  "old-temp",
+	FileTypeTemp:     "temp",
+	FileTypeBlob:     "blob",
+}
+
+// FileTypeFromName parses a FileType from its string representation.
+func FileTypeFromName(name string) FileType {
+	for i, s := range fileTypeStrings {
+		if s == name {
+			return FileType(i)
+		}
+	}
+	panic(fmt.Sprintf("unknown file type: %q", name))
+}
+
+// SafeFormat implements redact.SafeFormatter.
+func (ft FileType) SafeFormat(w redact.SafePrinter, _ rune) {
+	if ft < 0 || int(ft) >= len(fileTypeStrings) {
+		w.Print(redact.SafeString("unknown"))
+		return
+	}
+	w.Print(redact.SafeString(fileTypeStrings[ft]))
+}
+
+// String implements fmt.Stringer.
+func (ft FileType) String() string {
+	return redact.StringWithoutMarkers(ft)
+}
+
 // MakeFilename builds a filename from components.
 func MakeFilename(fileType FileType, dfn DiskFileNum) string {
 	switch fileType {
@@ -80,6 +116,8 @@ func MakeFilename(fileType FileType, dfn DiskFileNum) string {
 		return fmt.Sprintf("CURRENT.%s.dbtmp", dfn)
 	case FileTypeTemp:
 		return fmt.Sprintf("temporary.%s.dbtmp", dfn)
+	case FileTypeBlob:
+		return fmt.Sprintf("%s.blob", dfn)
 	}
 	panic("unreachable")
 }
@@ -130,10 +168,11 @@ func ParseFilename(fs vfs.FS, filename string) (fileType FileType, dfn DiskFileN
 		if !ok {
 			break
 		}
-		// TODO(sumeer): stop handling FileTypeLog in this function.
 		switch filename[i+1:] {
 		case "sst":
 			return FileTypeTable, dfn, true
+		case "blob":
+			return FileTypeBlob, dfn, true
 		}
 	}
 	return 0, dfn, false
diff --git a/internal/base/internal.go b/internal/base/internal.go
index 0c123eb4..bd6381bf 100644
--- a/internal/base/internal.go
+++ b/internal/base/internal.go
@@ -12,6 +12,8 @@ import (
 	"strings"
 	"sync/atomic"
 
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/redact"
 )
 
@@ -398,6 +400,9 @@ func (k InternalKey) EncodeTrailer() [8]byte {
 func (k InternalKey) Separator(
 	cmp Compare, sep Separator, buf []byte, other InternalKey,
 ) InternalKey {
+	if invariants.Enabled && (len(k.UserKey) == 0 || len(other.UserKey) == 0) {
+		panic(errors.AssertionFailedf("empty keys passed to Separator: %s, %s", k, other))
+	}
 	buf = sep(buf, k.UserKey, other.UserKey)
 	if len(buf) <= len(k.UserKey) && cmp(k.UserKey, buf) < 0 {
 		// The separator user key is physically shorter than k.UserKey (if it is
@@ -416,7 +421,7 @@ func (k InternalKey) Separator(
 // InternalKey.UserKey, though it is valid to pass a nil.
 func (k InternalKey) Successor(cmp Compare, succ Successor, buf []byte) InternalKey {
 	buf = succ(buf, k.UserKey)
-	if len(buf) <= len(k.UserKey) && cmp(k.UserKey, buf) < 0 {
+	if (len(k.UserKey) == 0 || len(buf) <= len(k.UserKey)) && cmp(k.UserKey, buf) < 0 {
 		// The successor user key is physically shorter that k.UserKey (if it is
 		// longer, we'll continue to use "k"), but logically after. Tack on the max
 		// sequence number to the shortened user key. Note that we could tack on
diff --git a/internal/cache/cache.go b/internal/cache/cache.go
new file mode 100644
index 00000000..68e370a9
--- /dev/null
+++ b/internal/cache/cache.go
@@ -0,0 +1,334 @@
+// Copyright 2025 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package cache
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"runtime"
+	"runtime/debug"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+)
+
+// Metrics holds metrics for the cache.
+type Metrics struct {
+	// The number of bytes inuse by the cache.
+	Size int64
+	// The count of objects (blocks or tables) in the cache.
+	Count int64
+	// The number of cache hits.
+	Hits int64
+	// The number of cache misses.
+	Misses int64
+}
+
+// Cache implements Pebble's sharded block cache. The Clock-PRO algorithm is
+// used for page replacement
+// (http://static.usenix.org/event/usenix05/tech/general/full_papers/jiang/jiang_html/html.html). In
+// order to provide better concurrency, 4 x NumCPUs shards are created, with
+// each shard being given 1/n of the target cache size. The Clock-PRO algorithm
+// is run independently on each shard.
+//
+// Blocks are keyed by an (handleID, fileNum, offset) triple. The handleID is a
+// namespace for file numbers and allows a single Cache to be shared between
+// multiple Pebble instances (via separate Handles). The fileNum and offset
+// refer to an sstable file number and the offset of the block within the file.
+// Because sstables are immutable and file numbers are never reused,
+// (fileNum,offset) are unique for the lifetime of a Pebble instance.
+//
+// In addition to maintaining a map from (fileNum,offset) to data, each shard
+// maintains a map of the cached blocks for a particular fileNum. This allows
+// efficient eviction of all blocks for a file (is used when an sstable is
+// deleted from disk).
+//
+// # Memory Management
+//
+// A normal implementation of the block cache would result in GC having to read
+// through all the structures and keep track of the liveness of many objects.
+// This was found to cause significant overhead in CRDB when compared to the
+// earlier use of RocksDB.
+//
+// In order to reduce pressure on the Go GC, manual memory management is
+// performed for the data stored in the cache. Manual memory management is
+// performed by calling into C.{malloc,free} to allocate memory; this memory is
+// outside the purview of the GC. Cache.Values are reference counted and the
+// memory backing a manual value is freed when the reference count drops to 0.
+//
+// Manual memory management brings the possibility of memory leaks. It is
+// imperative that every Handle returned by Cache.{Get,Set} is eventually
+// released. The "invariants" build tag enables a leak detection facility that
+// places a GC finalizer on cache.Value. When the cache.Value finalizer is run,
+// if the underlying buffer is still present a leak has occurred. The "tracing"
+// build tag enables tracing of cache.Value reference count manipulation and
+// eases finding where a leak has occurred. These two facilities are usually
+// used in combination by specifying `-tags invariants,tracing`. Note that
+// "tracing" produces a significant slowdown, while "invariants" does not.
+type Cache struct {
+	refs    atomic.Int64
+	maxSize int64
+	idAlloc atomic.Uint64
+	shards  []shard
+
+	// Traces recorded by Cache.trace. Used for debugging.
+	tr struct {
+		sync.Mutex
+		msgs []string
+	}
+	stack string
+}
+
+// New creates a new cache of the specified size. Memory for the cache is
+// allocated on demand, not during initialization. The cache is created with a
+// reference count of 1. Each DB it is associated with adds a reference, so the
+// creator of the cache should usually release their reference after the DB is
+// created.
+//
+//	c := cache.New(...)
+//	defer c.Unref()
+//	d, err := pebble.Open(pebble.Options{Cache: c})
+func New(size int64) *Cache {
+	// How many cache shards should we create?
+	//
+	// Note that the probability two processors will try to access the same
+	// shard at the same time increases superlinearly with the number of
+	// processors (consider the birthday problem where each CPU is a person,
+	// and each shard is a possible birthday).
+	//
+	// We could consider growing the number of shards superlinearly, but
+	// increasing the shard count may reduce the effectiveness of the caching
+	// algorithm if frequently-accessed blocks are insufficiently distributed
+	// across shards.
+	//
+	// Experimentally, we've observed contention contributing to tail latencies
+	// at 2 shards per processor. For now we use 4 shards per processor,
+	// recognizing this may not be final word.
+	m := 4 * runtime.GOMAXPROCS(0)
+
+	// In tests we can use large CPU machines with small cache sizes and have
+	// many caches in existence at a time. If sharding into m shards would
+	// produce too small shards, constrain the number of shards to 4.
+	const minimumShardSize = 4 << 20 // 4 MiB
+	if m > 4 && int(size)/m < minimumShardSize {
+		m = 4
+	}
+	return newCache(size, m)
+}
+
+func newCache(size int64, shards int) *Cache {
+	c := &Cache{
+		maxSize: size,
+		shards:  make([]shard, shards),
+		stack:   string(debug.Stack()),
+	}
+	c.refs.Store(1)
+	c.trace("alloc", c.refs.Load())
+	for i := range c.shards {
+		c.shards[i].init(size / int64(len(c.shards)))
+	}
+
+	// Note: this is a no-op if invariants are disabled or race is enabled.
+	invariants.SetFinalizer(c, func(c *Cache) {
+		if v := c.refs.Load(); v != 0 {
+			c.tr.Lock()
+			fmt.Fprintf(os.Stderr,
+				"pebble: cache (%p) has non-zero reference count: %d\n\n%s\n\n", c, v, c.stack)
+			if len(c.tr.msgs) > 0 {
+				fmt.Fprintf(os.Stderr, "%s\n", strings.Join(c.tr.msgs, "\n"))
+			}
+			c.tr.Unlock()
+			os.Exit(1)
+		}
+	})
+	return c
+}
+
+// Ref adds a reference to the cache. The cache only remains valid as long a
+// reference is maintained to it.
+func (c *Cache) Ref() {
+	v := c.refs.Add(1)
+	if v <= 1 {
+		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
+	}
+	c.trace("ref", v)
+}
+
+// Unref releases a reference on the cache.
+func (c *Cache) Unref() {
+	v := c.refs.Add(-1)
+	c.trace("unref", v)
+	switch {
+	case v < 0:
+		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
+	case v == 0:
+		for i := range c.shards {
+			c.shards[i].Free()
+		}
+	}
+}
+
+func (c *Cache) NewHandle() *Handle {
+	c.Ref()
+	id := handleID(c.idAlloc.Add(1))
+	return &Handle{
+		cache: c,
+		id:    id,
+	}
+}
+
+// Reserve N bytes in the cache. This effectively shrinks the size of the cache
+// by N bytes, without actually consuming any memory. The returned closure
+// should be invoked to release the reservation.
+func (c *Cache) Reserve(n int) func() {
+	// Round-up the per-shard reservation. Most reservations should be large, so
+	// this probably doesn't matter in practice.
+	shardN := (n + len(c.shards) - 1) / len(c.shards)
+	for i := range c.shards {
+		c.shards[i].Reserve(shardN)
+	}
+	return func() {
+		if shardN == -1 {
+			panic("pebble: cache reservation already released")
+		}
+		for i := range c.shards {
+			c.shards[i].Reserve(-shardN)
+		}
+		shardN = -1
+	}
+}
+
+// Metrics returns the metrics for the cache.
+func (c *Cache) Metrics() Metrics {
+	var m Metrics
+	for i := range c.shards {
+		s := &c.shards[i]
+		s.mu.RLock()
+		m.Count += int64(s.blocks.Len())
+		m.Size += s.sizeHot + s.sizeCold
+		s.mu.RUnlock()
+		m.Hits += s.hits.Load()
+		m.Misses += s.misses.Load()
+	}
+	return m
+}
+
+// MaxSize returns the max size of the cache.
+func (c *Cache) MaxSize() int64 {
+	return c.maxSize
+}
+
+// Size returns the current space used by the cache.
+func (c *Cache) Size() int64 {
+	var size int64
+	for i := range c.shards {
+		size += c.shards[i].Size()
+	}
+	return size
+}
+
+func (c *Cache) getShard(k key) *shard {
+	idx := k.shardIdx(len(c.shards))
+	return &c.shards[idx]
+}
+
+// Handle is the interface through which a store uses the cache. Each store uses
+// a separate "handle". A handle corresponds to a separate "namespace" inside
+// the cache; a handle cannot see another handle's blocks.
+type Handle struct {
+	cache *Cache
+	id    handleID
+}
+
+// handleID is an ID associated with a Handle; it is unique in the context of a
+// Cache instance and serves as a namespace for file numbers, allowing a single
+// Cache to be shared among multiple Pebble instances.
+type handleID uint64
+
+// Cache returns the Cache instance associated with the handle.
+func (c *Handle) Cache() *Cache {
+	return c.cache
+}
+
+// Get retrieves the cache value for the specified file and offset, returning
+// nil if no value is present.
+func (c *Handle) Get(fileNum base.DiskFileNum, offset uint64) *Value {
+	k := makeKey(c.id, fileNum, offset)
+	cv, re := c.cache.getShard(k).getWithMaybeReadEntry(k, false /* desireReadEntry */)
+	if invariants.Enabled && re != nil {
+		panic("readEntry should be nil")
+	}
+	return cv
+}
+
+// GetWithReadHandle retrieves the cache value for the specified handleID, fileNum
+// and offset. If found, a valid Handle is returned (with cacheHit set to
+// true), else a valid ReadHandle is returned.
+//
+// See the ReadHandle declaration for the contract the caller must satisfy
+// when getting a valid ReadHandle.
+//
+// This method can block before returning since multiple concurrent gets for
+// the same cache value will take turns getting a ReadHandle, which represents
+// permission to do the read. This blocking respects context cancellation, in
+// which case an error is returned (and not a valid ReadHandle).
+//
+// When blocking, the errorDuration return value can be non-zero and is
+// populated with the total duration that other readers that observed an error
+// (see ReadHandle.SetReadError) spent in doing the read. This duration can be
+// greater than the time spent blocked in this method, since some of these
+// errors could have occurred prior to this call. But it serves as a rough
+// indicator of whether turn taking could have caused higher latency due to
+// context cancellation of other readers.
+//
+// While waiting, someone else may successfully read the value, which results
+// in a valid Handle being returned. This is a case where cacheHit=false.
+func (c *Handle) GetWithReadHandle(
+	ctx context.Context, fileNum base.DiskFileNum, offset uint64,
+) (cv *Value, rh ReadHandle, errorDuration time.Duration, cacheHit bool, err error) {
+	k := makeKey(c.id, fileNum, offset)
+	cv, re := c.cache.getShard(k).getWithMaybeReadEntry(k, true /* desireReadEntry */)
+	if cv != nil {
+		return cv, ReadHandle{}, 0, true, nil
+	}
+	cv, errorDuration, err = re.waitForReadPermissionOrHandle(ctx)
+	if err != nil || cv != nil {
+		re.unrefAndTryRemoveFromMap()
+		return cv, ReadHandle{}, errorDuration, false, err
+	}
+	return nil, ReadHandle{entry: re}, errorDuration, false, nil
+}
+
+// Set sets the cache value for the specified file and offset, overwriting an
+// existing value if present. The value must have been allocated by Cache.Alloc.
+//
+// The cache takes a reference on the Value and holds it until it gets evicted.
+func (c *Handle) Set(fileNum base.DiskFileNum, offset uint64, value *Value) {
+	k := makeKey(c.id, fileNum, offset)
+	c.cache.getShard(k).set(k, value)
+}
+
+// Delete deletes the cached value for the specified file and offset.
+func (c *Handle) Delete(fileNum base.DiskFileNum, offset uint64) {
+	k := makeKey(c.id, fileNum, offset)
+	c.cache.getShard(k).delete(k)
+}
+
+// EvictFile evicts all cache values for the specified file.
+func (c *Handle) EvictFile(fileNum base.DiskFileNum) {
+	for i := range c.cache.shards {
+		c.cache.shards[i].evictFile(c.id, fileNum)
+	}
+}
+
+func (c *Handle) Close() {
+	c.cache.Unref()
+	*c = Handle{}
+}
diff --git a/internal/cache/clockpro.go b/internal/cache/clockpro.go
index ad860f06..cbd55d4b 100644
--- a/internal/cache/clockpro.go
+++ b/internal/cache/clockpro.go
@@ -18,15 +18,12 @@
 package cache // import "github.com/cockroachdb/pebble/internal/cache"
 
 import (
-	"context"
 	"fmt"
 	"os"
 	"runtime"
 	"runtime/debug"
-	"strings"
 	"sync"
 	"sync/atomic"
-	"time"
 
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
@@ -35,12 +32,12 @@ import (
 // key is associated with a specific block.
 type key struct {
 	// id is the namespace for fileNums.
-	id      ID
+	id      handleID
 	fileNum base.DiskFileNum
 	offset  uint64
 }
 
-func makeKey(id ID, fileNum base.DiskFileNum, offset uint64) key {
+func makeKey(id handleID, fileNum base.DiskFileNum, offset uint64) key {
 	return key{
 		id:      id,
 		fileNum: fileNum,
@@ -51,7 +48,7 @@ func makeKey(id ID, fileNum base.DiskFileNum, offset uint64) key {
 // shardIdx determines the shard index for the given key.
 func (k *key) shardIdx(numShards int) int {
 	if k.id == 0 {
-		panic("pebble: 0 cache ID is invalid")
+		panic("pebble: 0 cache handleID is invalid")
 	}
 	// Same as fibonacciHash() but without the cast to uintptr.
 	const m = 11400714819323198485
@@ -147,25 +144,11 @@ func (c *shard) getWithMaybeReadEntry(k key, desireReadEntry bool) (*Value, *rea
 			e.referenced.Store(true)
 		}
 	}
-	c.mu.RUnlock()
 	var re *readEntry
 	if value == nil && desireReadEntry {
-		c.mu.Lock()
-		// After the c.mu.RUnlock(), someone could have inserted the value in the
-		// cache. We could tolerate the race and do a file read, or do another map
-		// lookup. We choose to do the latter, since the cost of a map lookup is
-		// insignificant compared to the cost of reading a block from a file.
-		if e, _ := c.blocks.Get(k); e != nil {
-			value = e.acquireValue()
-			if value != nil {
-				e.referenced.Store(true)
-			}
-		}
-		if value == nil {
-			re = c.readShard.acquireReadEntryLocked(k)
-		}
-		c.mu.Unlock()
+		re = c.readShard.acquireReadEntry(k)
 	}
+	c.mu.RUnlock()
 	if value == nil {
 		c.misses.Add(1)
 	} else {
@@ -296,7 +279,7 @@ func (c *shard) delete(k key) {
 }
 
 // EvictFile evicts all of the cache values for the specified file.
-func (c *shard) evictFile(id ID, fileNum base.DiskFileNum) {
+func (c *shard) evictFile(id handleID, fileNum base.DiskFileNum) {
 	fkey := makeKey(id, fileNum, 0)
 	for c.evictFileRun(fkey) {
 		// Sched switch to give another goroutine an opportunity to acquire the
@@ -658,317 +641,3 @@ func (c *shard) runHandTest() {
 
 	c.handTest = c.handTest.next()
 }
-
-// Metrics holds metrics for the cache.
-type Metrics struct {
-	// The number of bytes inuse by the cache.
-	Size int64
-	// The count of objects (blocks or tables) in the cache.
-	Count int64
-	// The number of cache hits.
-	Hits int64
-	// The number of cache misses.
-	Misses int64
-}
-
-// Cache implements Pebble's sharded block cache. The Clock-PRO algorithm is
-// used for page replacement
-// (http://static.usenix.org/event/usenix05/tech/general/full_papers/jiang/jiang_html/html.html). In
-// order to provide better concurrency, 4 x NumCPUs shards are created, with
-// each shard being given 1/n of the target cache size. The Clock-PRO algorithm
-// is run independently on each shard.
-//
-// Blocks are keyed by an (id, fileNum, offset) triple. The ID is a namespace
-// for file numbers and allows a single Cache to be shared between multiple
-// Pebble instances. The fileNum and offset refer to an sstable file number and
-// the offset of the block within the file. Because sstables are immutable and
-// file numbers are never reused, (fileNum,offset) are unique for the lifetime
-// of a Pebble instance.
-//
-// In addition to maintaining a map from (fileNum,offset) to data, each shard
-// maintains a map of the cached blocks for a particular fileNum. This allows
-// efficient eviction of all of the blocks for a file which is used when an
-// sstable is deleted from disk.
-//
-// # Memory Management
-//
-// A normal implementation of the block cache would result in GC having to read
-// through all the structures and keep track of the liveness of many objects.
-// This was found to cause significant overhead in CRDB when compared to the
-// earlier use of RocksDB.
-//
-// In order to reduce pressure on the Go GC, manual memory management is
-// performed for the data stored in the cache. Manual memory management is
-// performed by calling into C.{malloc,free} to allocate memory; this memory is
-// outside the purview of the GC. Cache.Values are reference counted and the
-// memory backing a manual value is freed when the reference count drops to 0.
-//
-// Manual memory management brings the possibility of memory leaks. It is
-// imperative that every Handle returned by Cache.{Get,Set} is eventually
-// released. The "invariants" build tag enables a leak detection facility that
-// places a GC finalizer on cache.Value. When the cache.Value finalizer is run,
-// if the underlying buffer is still present a leak has occurred. The "tracing"
-// build tag enables tracing of cache.Value reference count manipulation and
-// eases finding where a leak has occurred. These two facilities are usually
-// used in combination by specifying `-tags invariants,tracing`. Note that
-// "tracing" produces a significant slowdown, while "invariants" does not.
-type Cache struct {
-	refs    atomic.Int64
-	maxSize int64
-	idAlloc atomic.Uint64
-	shards  []shard
-
-	// Traces recorded by Cache.trace. Used for debugging.
-	tr struct {
-		sync.Mutex
-		msgs []string
-	}
-}
-
-// ID is a namespace for file numbers. It allows a single Cache to be shared
-// among multiple Pebble instances. NewID can be used to generate a new ID that
-// is unique in the context of this cache.
-type ID uint64
-
-// New creates a new cache of the specified size. Memory for the cache is
-// allocated on demand, not during initialization. The cache is created with a
-// reference count of 1. Each DB it is associated with adds a reference, so the
-// creator of the cache should usually release their reference after the DB is
-// created.
-//
-//	c := cache.New(...)
-//	defer c.Unref()
-//	d, err := pebble.Open(pebble.Options{Cache: c})
-func New(size int64) *Cache {
-	// How many cache shards should we create?
-	//
-	// Note that the probability two processors will try to access the same
-	// shard at the same time increases superlinearly with the number of
-	// processors (consider the birthday problem where each CPU is a person,
-	// and each shard is a possible birthday).
-	//
-	// We could consider growing the number of shards superlinearly, but
-	// increasing the shard count may reduce the effectiveness of the caching
-	// algorithm if frequently-accessed blocks are insufficiently distributed
-	// across shards.
-	//
-	// Experimentally, we've observed contention contributing to tail latencies
-	// at 2 shards per processor. For now we use 4 shards per processor,
-	// recognizing this may not be final word.
-	m := 4 * runtime.GOMAXPROCS(0)
-
-	// In tests we can use large CPU machines with small cache sizes and have
-	// many caches in existence at a time. If sharding into m shards would
-	// produce too small shards, constrain the number of shards to 4.
-	const minimumShardSize = 4 << 20 // 4 MiB
-	if m > 4 && int(size)/m < minimumShardSize {
-		m = 4
-	}
-	return newShards(size, m)
-}
-
-func newShards(size int64, shards int) *Cache {
-	c := &Cache{
-		maxSize: size,
-		shards:  make([]shard, shards),
-	}
-	c.refs.Store(1)
-	c.idAlloc.Store(1)
-	c.trace("alloc", c.refs.Load())
-	for i := range c.shards {
-		c.shards[i].init(size / int64(len(c.shards)))
-	}
-
-	// Note: this is a no-op if invariants are disabled or race is enabled.
-	invariants.SetFinalizer(c, func(c *Cache) {
-		if v := c.refs.Load(); v != 0 {
-			c.tr.Lock()
-			fmt.Fprintf(os.Stderr,
-				"pebble: cache (%p) has non-zero reference count: %d\n", c, v)
-			if len(c.tr.msgs) > 0 {
-				fmt.Fprintf(os.Stderr, "%s\n", strings.Join(c.tr.msgs, "\n"))
-			}
-			c.tr.Unlock()
-			os.Exit(1)
-		}
-	})
-	return c
-}
-
-func (c *Cache) getShard(k key) *shard {
-	idx := k.shardIdx(len(c.shards))
-	return &c.shards[idx]
-}
-
-// Ref adds a reference to the cache. The cache only remains valid as long a
-// reference is maintained to it.
-func (c *Cache) Ref() {
-	v := c.refs.Add(1)
-	if v <= 1 {
-		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
-	}
-	c.trace("ref", v)
-}
-
-// Unref releases a reference on the cache.
-func (c *Cache) Unref() {
-	v := c.refs.Add(-1)
-	c.trace("unref", v)
-	switch {
-	case v < 0:
-		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
-	case v == 0:
-		for i := range c.shards {
-			c.shards[i].Free()
-		}
-	}
-}
-
-// Get retrieves the cache value for the specified file and offset, returning
-// nil if no value is present.
-func (c *Cache) Get(id ID, fileNum base.DiskFileNum, offset uint64) *Value {
-	k := makeKey(id, fileNum, offset)
-	cv, re := c.getShard(k).getWithMaybeReadEntry(k, false /* desireReadEntry */)
-	if invariants.Enabled && re != nil {
-		panic("readEntry should be nil")
-	}
-	return cv
-}
-
-// GetWithReadHandle retrieves the cache value for the specified ID, fileNum
-// and offset. If found, a valid Handle is returned (with cacheHit set to
-// true), else a valid ReadHandle is returned.
-//
-// See the ReadHandle declaration for the contract the caller must satisfy
-// when getting a valid ReadHandle.
-//
-// This method can block before returning since multiple concurrent gets for
-// the same cache value will take turns getting a ReadHandle, which represents
-// permission to do the read. This blocking respects context cancellation, in
-// which case an error is returned (and not a valid ReadHandle).
-//
-// When blocking, the errorDuration return value can be non-zero and is
-// populated with the total duration that other readers that observed an error
-// (see ReadHandle.SetReadError) spent in doing the read. This duration can be
-// greater than the time spent blocked in this method, since some of these
-// errors could have occurred prior to this call. But it serves as a rough
-// indicator of whether turn taking could have caused higher latency due to
-// context cancellation of other readers.
-//
-// While waiting, someone else may successfully read the value, which results
-// in a valid Handle being returned. This is a case where cacheHit=false.
-func (c *Cache) GetWithReadHandle(
-	ctx context.Context, id ID, fileNum base.DiskFileNum, offset uint64,
-) (cv *Value, rh ReadHandle, errorDuration time.Duration, cacheHit bool, err error) {
-	k := makeKey(id, fileNum, offset)
-	cv, re := c.getShard(k).getWithMaybeReadEntry(k, true /* desireReadEntry */)
-	if cv != nil {
-		return cv, ReadHandle{}, 0, true, nil
-	}
-	cv, errorDuration, err = re.waitForReadPermissionOrHandle(ctx)
-	if err != nil || cv != nil {
-		re.unrefAndTryRemoveFromMap()
-		return cv, ReadHandle{}, errorDuration, false, err
-	}
-	return nil, ReadHandle{entry: re}, errorDuration, false, nil
-}
-
-// Set sets the cache value for the specified file and offset, overwriting an
-// existing value if present. The value must have been allocated by Cache.Alloc.
-//
-// The cache takes a reference on the Value and holds it until it gets evicted.
-func (c *Cache) Set(id ID, fileNum base.DiskFileNum, offset uint64, value *Value) {
-	k := makeKey(id, fileNum, offset)
-	c.getShard(k).set(k, value)
-}
-
-// Delete deletes the cached value for the specified file and offset.
-func (c *Cache) Delete(id ID, fileNum base.DiskFileNum, offset uint64) {
-	k := makeKey(id, fileNum, offset)
-	c.getShard(k).delete(k)
-}
-
-// EvictFile evicts all of the cache values for the specified file.
-func (c *Cache) EvictFile(id ID, fileNum base.DiskFileNum) {
-	if id == 0 {
-		panic("pebble: 0 cache ID is invalid")
-	}
-	for i := range c.shards {
-		c.shards[i].evictFile(id, fileNum)
-	}
-}
-
-// MaxSize returns the max size of the cache.
-func (c *Cache) MaxSize() int64 {
-	return c.maxSize
-}
-
-// Size returns the current space used by the cache.
-func (c *Cache) Size() int64 {
-	var size int64
-	for i := range c.shards {
-		size += c.shards[i].Size()
-	}
-	return size
-}
-
-// Alloc allocates a byte slice of the specified size, possibly reusing
-// previously allocated but unused memory. The memory backing the value is
-// manually managed. The caller MUST either add the value to the cache (via
-// Cache.Set), or release the value (via Cache.Free). Failure to do so will
-// result in a memory leak.
-func Alloc(n int) *Value {
-	return newValue(n)
-}
-
-// Free frees the specified value. The buffer associated with the value will
-// possibly be reused, making it invalid to use the buffer after calling
-// Free. Do not call Free on a value that has been added to the cache.
-func Free(v *Value) {
-	if n := v.refs(); n > 1 {
-		panic(fmt.Sprintf("pebble: Value has been added to the cache: refs=%d", n))
-	}
-	v.Release()
-}
-
-// Reserve N bytes in the cache. This effectively shrinks the size of the cache
-// by N bytes, without actually consuming any memory. The returned closure
-// should be invoked to release the reservation.
-func (c *Cache) Reserve(n int) func() {
-	// Round-up the per-shard reservation. Most reservations should be large, so
-	// this probably doesn't matter in practice.
-	shardN := (n + len(c.shards) - 1) / len(c.shards)
-	for i := range c.shards {
-		c.shards[i].Reserve(shardN)
-	}
-	return func() {
-		if shardN == -1 {
-			panic("pebble: cache reservation already released")
-		}
-		for i := range c.shards {
-			c.shards[i].Reserve(-shardN)
-		}
-		shardN = -1
-	}
-}
-
-// Metrics returns the metrics for the cache.
-func (c *Cache) Metrics() Metrics {
-	var m Metrics
-	for i := range c.shards {
-		s := &c.shards[i]
-		s.mu.RLock()
-		m.Count += int64(s.blocks.Len())
-		m.Size += s.sizeHot + s.sizeCold
-		s.mu.RUnlock()
-		m.Hits += s.hits.Load()
-		m.Misses += s.misses.Load()
-	}
-	return m
-}
-
-// NewID returns a new ID to be used as a namespace for cached file
-// blocks.
-func (c *Cache) NewID() ID {
-	return ID(c.idAlloc.Add(1))
-}
diff --git a/internal/cache/clockpro_normal.go b/internal/cache/clockpro_normal.go
index ae49938e..c8b250c6 100644
--- a/internal/cache/clockpro_normal.go
+++ b/internal/cache/clockpro_normal.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build !tracing
-// +build !tracing
 
 package cache
 
diff --git a/internal/cache/clockpro_tracing.go b/internal/cache/clockpro_tracing.go
index d14c1cdc..ccc592b3 100644
--- a/internal/cache/clockpro_tracing.go
+++ b/internal/cache/clockpro_tracing.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build tracing
-// +build tracing
 
 package cache
 
diff --git a/internal/cache/read_shard.go b/internal/cache/read_shard.go
index 5a1328c9..0f70c5e3 100644
--- a/internal/cache/read_shard.go
+++ b/internal/cache/read_shard.go
@@ -9,6 +9,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/swiss"
 )
 
@@ -39,16 +40,15 @@ import (
 //
 // Design choices and motivation:
 //
-//   - readShard is tightly integrated with a cache shard: At its core,
-//     readShard is a map with synchronization. For the same reason the cache is
-//     sharded (for higher concurrency by sharding the mutex), it is beneficial
-//     to shard synchronization on readShard. By making readShard a member of
-//     shard, this sharding is trivially accomplished. Additionally, the code
-//     feels cleaner when there isn't a race between a cache miss, followed by
-//     creating a readEntry that is no longer needed because someone else has
-//     done the read since the miss and inserted into the cache. By making the
-//     readShard use shard.mu, such a race is avoided. A side benefit is that
-//     the cache interaction can be hidden behind readEntry.SetReadValue. One
+//   - At its core, readShard is a map with synchronization. For the same reason
+//     the cache is sharded (for higher concurrency by sharding the mutex), it
+//     is beneficial to shard synchronization on readShard. By making readShard
+//     a member of shard, this sharding is trivially accomplished. readShard has
+//     its own mutex (separate from shard.mu), in order to avoid write-locking
+//     shard.mu when we start a read.
+//
+//   - readShard is integrated with the corresponding cache shard; this allows
+//     the cache interaction to be hidden behind readEntry.SetReadValue. One
 //     disadvantage of this tightly integrated design is that it does not
 //     encompass readers that will put the read value into a block.BufferPool --
 //     we don't worry about those since block.BufferPool is only used for
@@ -69,10 +69,8 @@ type readShard struct {
 	// shard is only used for locking, and calling shard.Set.
 	shard *shard
 	// Protected by shard.mu.
-	//
-	// shard.mu is never held when acquiring readEntry.mu. shard.mu is a shared
-	// resource and must be released quickly.
-	shardMu struct {
+	mu struct {
+		sync.Mutex
 		readMap swiss.Map[key, *readEntry]
 	}
 }
@@ -82,27 +80,35 @@ func (rs *readShard) Init(shard *shard) *readShard {
 		shard: shard,
 	}
 	// Choice of 16 is arbitrary.
-	rs.shardMu.readMap.Init(16)
+	rs.mu.readMap.Init(16)
 	return rs
 }
 
-// acquireReadEntryLocked gets a *readEntry for (id, fileNum, offset). shard.mu is
-// already write locked.
-func (rs *readShard) acquireReadEntryLocked(k key) *readEntry {
-	e, ok := rs.shardMu.readMap.Get(k)
-	if !ok {
-		e = newReadEntry(rs, k)
-		rs.shardMu.readMap.Put(k, e)
-	} else {
-		e.refCount.acquireAllowZero()
+// acquireReadEntry acquires a *readEntry for (id, fileNum, offset), creating
+// one if necessary.
+func (rs *readShard) acquireReadEntry(k key) *readEntry {
+	rs.mu.Lock()
+	defer rs.mu.Unlock()
+
+	if e, ok := rs.mu.readMap.Get(k); ok {
+		// An entry we found in the map while holding the mutex must have a non-zero
+		// reference count.
+		if e.refCount < 1 {
+			panic("invalid reference count")
+		}
+		e.refCount++
+		return e
 	}
+
+	e := newReadEntry(rs, k)
+	rs.mu.readMap.Put(k, e)
 	return e
 }
 
 func (rs *readShard) lenForTesting() int {
-	rs.shard.mu.Lock()
-	defer rs.shard.mu.Unlock()
-	return rs.shardMu.readMap.Len()
+	rs.mu.Lock()
+	defer rs.mu.Unlock()
+	return rs.mu.readMap.Len()
 }
 
 // readEntry is used to coordinate between concurrent attempted readers of the
@@ -146,10 +152,8 @@ type readEntry struct {
 		errorDuration time.Duration
 		readStart     time.Time
 	}
-	// Count of ReadHandles that refer to this readEntry. Increments always hold
-	// shard.mu. So if this is found to be 0 while holding shard.mu, it is safe
-	// to delete readEntry from readShard.shardMu.readMap.
-	refCount refcnt
+	// Count of ReadHandles that refer to this readEntry. Protected by readShard.mu.
+	refCount int32
 }
 
 var readEntryPool = sync.Pool{
@@ -163,8 +167,8 @@ func newReadEntry(rs *readShard, k key) *readEntry {
 	*e = readEntry{
 		readShard: rs,
 		key:       k,
+		refCount:  1,
 	}
-	e.refCount.init(1)
 	return e
 }
 
@@ -261,40 +265,26 @@ func (e *readEntry) waitForReadPermissionOrHandle(
 
 // unrefAndTryRemoveFromMap reduces the reference count of e and removes e.key
 // => e from the readMap if necessary.
-//
-// It is possible that after unreffing that s.e has already been removed, and
-// is now back in the sync.Pool, or being reused (for the same or different
-// key). This is because after unreffing, which caused the s.e.refCount to
-// become zero, but before acquiring shard.mu, it could have been incremented
-// and decremented concurrently, and some other goroutine could have observed
-// a different decrement to 0, and raced ahead and deleted s.e from the
-// readMap.
 func (e *readEntry) unrefAndTryRemoveFromMap() {
-	// Save the fields we need from entry; once we release the last refcount, it
-	// is possible that the entry is found and reused and then freed.
 	rs := e.readShard
-	k := e.key
-	if !e.refCount.release() {
+	rs.mu.Lock()
+	e.refCount--
+	if e.refCount > 0 {
+		// Entry still in use.
+		rs.mu.Unlock()
 		return
 	}
-	// Once we release the refcount, it is possible that it the entry is reused
-	// again and freed before we get the lock.
-	rs.shard.mu.Lock()
-	e2, ok := rs.shardMu.readMap.Get(k)
-	if !ok || e2 != e {
-		// Already removed.
-		rs.shard.mu.Unlock()
-		return
+	if e.refCount < 0 {
+		panic("invalid reference count")
 	}
-	if e.refCount.value() != 0 {
-		// The entry was reused.
-		rs.shard.mu.Unlock()
-		return
+	// The refcount is now 0; remove from the map.
+	if invariants.Enabled {
+		if e2, ok := rs.mu.readMap.Get(e.key); !ok || e2 != e {
+			panic("entry not in readMap")
+		}
 	}
-	// e.refCount == 0. And it cannot be incremented since
-	// shard.mu.Lock() is held. So remove from map.
-	rs.shardMu.readMap.Delete(k)
-	rs.shard.mu.Unlock()
+	rs.mu.readMap.Delete(e.key)
+	rs.mu.Unlock()
 
 	// Free s.e.
 	e.mu.v.Release()
diff --git a/internal/cache/refcnt_normal.go b/internal/cache/refcnt_normal.go
index bf1a4756..a1592672 100644
--- a/internal/cache/refcnt_normal.go
+++ b/internal/cache/refcnt_normal.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build !tracing
-// +build !tracing
 
 package cache
 
@@ -37,15 +36,8 @@ func (v *refcnt) acquire() {
 	}
 }
 
-// acquireAllowZero is the same as acquire, but allows acquireAllowZero to be
-// called with a zero refcnt. This is useful for cases where the entry which
-// is being reference counted is inside a container and the container does not
-// hold a reference. The container uses release() returning true to attempt to
-// do a cleanup from the container.
-func (v *refcnt) acquireAllowZero() {
-	v.val.Add(1)
-}
-
+// release decrements the reference count and returns true when the reference
+// count becomes 0.
 func (v *refcnt) release() bool {
 	switch v := v.val.Add(-1); {
 	case v < 0:
@@ -57,10 +49,6 @@ func (v *refcnt) release() bool {
 	}
 }
 
-func (v *refcnt) value() int32 {
-	return v.val.Load()
-}
-
 func (v *refcnt) trace(msg string) {
 }
 
diff --git a/internal/cache/refcnt_tracing.go b/internal/cache/refcnt_tracing.go
index dd94088e..151c68d0 100644
--- a/internal/cache/refcnt_tracing.go
+++ b/internal/cache/refcnt_tracing.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build tracing
-// +build tracing
 
 package cache
 
@@ -41,16 +40,6 @@ func (v *refcnt) acquire() {
 	v.trace("acquire")
 }
 
-// acquireAllowZero is the same as acquire, but allows acquireAllowZero to be
-// called with a zero refcnt. This is useful for cases where the entry which
-// is being reference counted is inside a container and the container does not
-// hold a reference. The container uses release() returning true to attempt to
-// do a cleanup from the container.
-func (v *refcnt) acquireAllowZero() {
-	v.val.Add(1)
-	v.trace("acquire")
-}
-
 func (v *refcnt) release() bool {
 	n := v.val.Add(-1)
 	switch {
@@ -61,10 +50,6 @@ func (v *refcnt) release() bool {
 	return n == 0
 }
 
-func (v *refcnt) value() int32 {
-	return v.val.Load()
-}
-
 func (v *refcnt) trace(msg string) {
 	s := fmt.Sprintf("%s: refs=%d\n%s", msg, v.refs(), debug.Stack())
 	v.Lock()
diff --git a/internal/cache/value.go b/internal/cache/value.go
index 82655bc0..d025d2a9 100644
--- a/internal/cache/value.go
+++ b/internal/cache/value.go
@@ -41,7 +41,12 @@ type Value struct {
 // want to add finalizer assertions.
 const valueEntryGoAllocated = !buildtags.Cgo || invariants.UseFinalizers
 
-func newValue(n int) *Value {
+// Alloc allocates a byte slice of the specified size, possibly reusing
+// previously allocated but unused memory. The memory backing the value is
+// manually managed. The caller MUST either add the value to the cache (via
+// Cache.Set), or release the value (via Cache.Free). Failure to do so will
+// result in a memory leak.
+func Alloc(n int) *Value {
 	if n == 0 {
 		return nil
 	}
@@ -126,3 +131,13 @@ func (v *Value) Release() {
 		v.free()
 	}
 }
+
+// Free frees the specified value. The buffer associated with the value will
+// possibly be reused, making it invalid to use the buffer after calling
+// Free. Do not call Free on a value that has been added to the cache.
+func Free(v *Value) {
+	if n := v.refs(); n > 1 {
+		panic(fmt.Sprintf("pebble: Value has been added to the cache: refs=%d", n))
+	}
+	v.Release()
+}
diff --git a/internal/devtools/tools.go b/internal/devtools/tools.go
index 132cc746..516cd9d2 100644
--- a/internal/devtools/tools.go
+++ b/internal/devtools/tools.go
@@ -1,5 +1,4 @@
 //go:build tools
-// +build tools
 
 package tools
 
diff --git a/internal/manifest/blob_metadata.go b/internal/manifest/blob_metadata.go
new file mode 100644
index 00000000..7559fdc5
--- /dev/null
+++ b/internal/manifest/blob_metadata.go
@@ -0,0 +1,93 @@
+// Copyright 2025 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package manifest
+
+import (
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/humanize"
+	"github.com/cockroachdb/redact"
+)
+
+// A BlobReference describes a sstable's reference to a blob value file.
+type BlobReference struct {
+	FileNum   base.DiskFileNum
+	ValueSize uint64
+}
+
+// BlobFileMetadata is metadata describing a blob value file.
+type BlobFileMetadata struct {
+	// FileNum is the file number.
+	FileNum base.DiskFileNum
+	// Size is the size of the file, in bytes.
+	Size uint64
+	// ValueSize is the sum of the length of the uncompressed values stored in
+	// this blob file.
+	ValueSize uint64
+	// File creation time in seconds since the epoch (1970-01-01 00:00:00
+	// UTC).
+	CreationTime uint64
+}
+
+// SafeFormat implements redact.SafeFormatter.
+func (m *BlobFileMetadata) SafeFormat(w redact.SafePrinter, _ rune) {
+	w.Printf("%s size:[%d (%s)] vals:[%d (%s)]",
+		m.FileNum, redact.Safe(m.Size), humanize.Bytes.Uint64(m.Size),
+		redact.Safe(m.ValueSize), humanize.Bytes.Uint64(m.ValueSize))
+}
+
+// String implements fmt.Stringer.
+func (m *BlobFileMetadata) String() string {
+	return redact.StringWithoutMarkers(m)
+}
+
+// ParseBlobFileMetadataDebug parses a BlobFileMetadata from its string
+// representation. This function is intended for use in tests. It's the inverse
+// of BlobFileMetadata.String().
+//
+// In production code paths, the BlobFileMetadata is serialized in a binary
+// format within a version edit under the tag tagNewBlobFile.
+func ParseBlobFileMetadataDebug(s string) (_ *BlobFileMetadata, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = errors.CombineErrors(err, errFromPanic(r))
+		}
+	}()
+
+	// Input format:
+	//  000000: size:[206536 (201KiB)], vals:[393256 (384KiB)]
+	m := &BlobFileMetadata{}
+	p := makeDebugParser(s)
+	m.FileNum = base.DiskFileNum(p.FileNum())
+
+	maybeSkipParens := func() {
+		if p.Peek() != "(" {
+			return
+		}
+		for p.Next() != ")" {
+			// Skip.
+		}
+	}
+
+	for !p.Done() {
+		field := p.Next()
+		p.Expect(":")
+		switch field {
+		case "size":
+			p.Expect("[")
+			m.Size = p.Uint64()
+			maybeSkipParens()
+			p.Expect("]")
+		case "vals":
+			p.Expect("[")
+			m.ValueSize = p.Uint64()
+			maybeSkipParens()
+			p.Expect("]")
+		default:
+			p.Errf("unknown field %q", field)
+		}
+	}
+	return m, nil
+}
diff --git a/internal/manifest/version.go b/internal/manifest/version.go
index 3af2fa0b..61f9177e 100644
--- a/internal/manifest/version.go
+++ b/internal/manifest/version.go
@@ -234,6 +234,10 @@ type FileMetadata struct {
 	// and are updated via the MaybeExtend{Point,Range}KeyBounds methods.
 	Smallest InternalKey
 	Largest  InternalKey
+	// BlobReferences is a list of blob files containing values that are
+	// referenced by this sstable.
+	BlobReferences []BlobReference
+
 	// Stats describe table statistics. Protected by DB.mu.
 	//
 	// For virtual sstables, set stats upon virtual sstable creation as
@@ -777,6 +781,16 @@ func (m *FileMetadata) DebugString(format base.FormatKey, verbose bool) string {
 	if m.Size != 0 {
 		fmt.Fprintf(&b, " size:%d", m.Size)
 	}
+	if len(m.BlobReferences) > 0 {
+		fmt.Fprintf(&b, " blobrefs:[")
+		for i, r := range m.BlobReferences {
+			if i > 0 {
+				fmt.Fprint(&b, ", ")
+			}
+			fmt.Fprintf(&b, "(%s: %d)", r.FileNum, r.ValueSize)
+		}
+		fmt.Fprint(&b, "]")
+	}
 	return b.String()
 }
 
@@ -837,6 +851,22 @@ func ParseFileMetadataDebug(s string) (_ *FileMetadata, err error) {
 		case "size":
 			m.Size = p.Uint64()
 
+		case "blobrefs":
+			p.Expect("[")
+			for p.Peek() != "]" {
+				if p.Peek() == "," {
+					p.Expect(",")
+				}
+				p.Expect("(")
+				var ref BlobReference
+				ref.FileNum = base.DiskFileNum(p.FileNum())
+				p.Expect(":")
+				ref.ValueSize = p.Uint64()
+				m.BlobReferences = append(m.BlobReferences, ref)
+				p.Expect(")")
+			}
+			p.Expect("]")
+
 		default:
 			p.Errf("unknown field %q", field)
 		}
diff --git a/internal/manifest/version_edit.go b/internal/manifest/version_edit.go
index 28b2799b..79e7ee58 100644
--- a/internal/manifest/version_edit.go
+++ b/internal/manifest/version_edit.go
@@ -57,6 +57,8 @@ const (
 	tagNewFile5            = 104 // Range keys.
 	tagCreatedBackingTable = 105
 	tagRemovedBackingTable = 106
+	tagNewBlobFile         = 107
+	tagDeletedBlobFile     = 108
 
 	// The custom tags sub-format used by tagNewFile4 and above. All tags less
 	// than customTagNonSafeIgnoreMask are safe to ignore and their format must be
@@ -69,6 +71,7 @@ const (
 	customTagVirtual           = 66
 	customTagSyntheticPrefix   = 67
 	customTagSyntheticSuffix   = 68
+	customTagBlobReferences    = 69
 )
 
 // DeletedFileEntry holds the state for a file deletion from a level. The file
@@ -152,6 +155,13 @@ type VersionEdit struct {
 	// and RemovedBackingTables. A file must be present in RemovedBackingTables
 	// in exactly one version edit.
 	RemovedBackingTables []base.DiskFileNum
+	// NewBlobFiles holds the metadata for all new blob files introduced within
+	// the version edit.
+	NewBlobFiles []*BlobFileMetadata
+	// DeletedBlobFiles holds all blob files that became unreferenced during the
+	// version edit. These blob files must not be referenced by any sstable in
+	// the resulting Version.
+	DeletedBlobFiles []base.DiskFileNum
 }
 
 // Decode decodes an edit from the specified reader.
@@ -346,6 +356,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 			}{}
 			var syntheticPrefix sstable.SyntheticPrefix
 			var syntheticSuffix sstable.SyntheticSuffix
+			var blobReferences []BlobReference
 			if tag == tagNewFile4 || tag == tagNewFile5 {
 				for {
 					customTag, err := d.readUvarint()
@@ -398,6 +409,28 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 							return err
 						}
 
+					case customTagBlobReferences:
+						n, err := d.readUvarint()
+						if err != nil {
+							return err
+						}
+						blobReferences = make([]BlobReference, n)
+						for i := 0; i < int(n); i++ {
+							fileNum, err := d.readUvarint()
+							if err != nil {
+								return err
+							}
+							valueSize, err := d.readUvarint()
+							if err != nil {
+								return err
+							}
+							blobReferences[i] = BlobReference{
+								FileNum:   base.DiskFileNum(fileNum),
+								ValueSize: valueSize,
+							}
+						}
+						continue
+
 					default:
 						if (customTag & customTagNonSafeIgnoreMask) != 0 {
 							return base.CorruptionErrorf("new-file4: custom field not supported: %d", customTag)
@@ -415,6 +448,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 				SmallestSeqNum:           smallestSeqNum,
 				LargestSeqNum:            largestSeqNum,
 				LargestSeqNumAbsolute:    largestSeqNum,
+				BlobReferences:           blobReferences,
 				MarkedForCompaction:      markedForCompaction,
 				Virtual:                  virtualState.virtual,
 				SyntheticPrefixAndSuffix: sstable.MakeSyntheticPrefixAndSuffix(syntheticPrefix, syntheticSuffix),
@@ -462,6 +496,37 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 			}
 			v.NewFiles = append(v.NewFiles, nfe)
 
+		case tagNewBlobFile:
+			fileNum, err := d.readFileNum()
+			if err != nil {
+				return err
+			}
+			size, err := d.readUvarint()
+			if err != nil {
+				return err
+			}
+			valueSize, err := d.readUvarint()
+			if err != nil {
+				return err
+			}
+			creationTime, err := d.readUvarint()
+			if err != nil {
+				return err
+			}
+			v.NewBlobFiles = append(v.NewBlobFiles, &BlobFileMetadata{
+				FileNum:      base.DiskFileNum(fileNum),
+				Size:         size,
+				ValueSize:    valueSize,
+				CreationTime: creationTime,
+			})
+
+		case tagDeletedBlobFile:
+			fileNum, err := d.readFileNum()
+			if err != nil {
+				return err
+			}
+			v.DeletedBlobFiles = append(v.DeletedBlobFiles, base.DiskFileNum(fileNum))
+
 		case tagPrevLogNumber:
 			n, err := d.readUvarint()
 			if err != nil {
@@ -525,6 +590,12 @@ func (v *VersionEdit) string(verbose bool, fmtKey base.FormatKey) string {
 	for _, n := range v.RemovedBackingTables {
 		fmt.Fprintf(&buf, "  del-backing:   %s\n", n)
 	}
+	for _, f := range v.NewBlobFiles {
+		fmt.Fprintf(&buf, "  add-blob-file: %s\n", f.String())
+	}
+	for _, df := range v.DeletedBlobFiles {
+		fmt.Fprintf(&buf, "  del-blob-file: %s\n", df)
+	}
 	return buf.String()
 }
 
@@ -596,6 +667,16 @@ func ParseVersionEditDebug(s string) (_ *VersionEdit, err error) {
 			n := p.DiskFileNum()
 			ve.RemovedBackingTables = append(ve.RemovedBackingTables, n)
 
+		case "add-blob-file":
+			meta, err := ParseBlobFileMetadataDebug(p.Remaining())
+			if err != nil {
+				return nil, err
+			}
+			ve.NewBlobFiles = append(ve.NewBlobFiles, meta)
+
+		case "del-blob-file":
+			ve.DeletedBlobFiles = append(ve.DeletedBlobFiles, p.DiskFileNum())
+
 		default:
 			return nil, errors.Errorf("field %q not implemented", field)
 		}
@@ -645,7 +726,7 @@ func (v *VersionEdit) Encode(w io.Writer) error {
 		e.writeUvarint(uint64(x.FileNum))
 	}
 	for _, x := range v.NewFiles {
-		customFields := x.Meta.MarkedForCompaction || x.Meta.CreationTime != 0 || x.Meta.Virtual
+		customFields := x.Meta.MarkedForCompaction || x.Meta.CreationTime != 0 || x.Meta.Virtual || len(x.Meta.BlobReferences) > 0
 		var tag uint64
 		switch {
 		case x.Meta.HasRangeKeys:
@@ -710,9 +791,28 @@ func (v *VersionEdit) Encode(w io.Writer) error {
 				e.writeUvarint(customTagSyntheticSuffix)
 				e.writeBytes(x.Meta.SyntheticPrefixAndSuffix.Suffix())
 			}
+			if len(x.Meta.BlobReferences) > 0 {
+				e.writeUvarint(customTagBlobReferences)
+				e.writeUvarint(uint64(len(x.Meta.BlobReferences)))
+				for _, ref := range x.Meta.BlobReferences {
+					e.writeUvarint(uint64(ref.FileNum))
+					e.writeUvarint(ref.ValueSize)
+				}
+			}
 			e.writeUvarint(customTagTerminate)
 		}
 	}
+	for _, x := range v.NewBlobFiles {
+		e.writeUvarint(tagNewBlobFile)
+		e.writeUvarint(uint64(x.FileNum))
+		e.writeUvarint(x.Size)
+		e.writeUvarint(x.ValueSize)
+		e.writeUvarint(x.CreationTime)
+	}
+	for _, x := range v.DeletedBlobFiles {
+		e.writeUvarint(tagDeletedBlobFile)
+		e.writeUvarint(uint64(x))
+	}
 	_, err := w.Write(e.Bytes())
 	return err
 }
diff --git a/internal/manual/manual_nocgo.go b/internal/manual/manual_nocgo.go
index 5a732b3c..5c38f674 100644
--- a/internal/manual/manual_nocgo.go
+++ b/internal/manual/manual_nocgo.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build !cgo
-// +build !cgo
 
 package manual
 
diff --git a/internal/metamorphic/metaflags/meta_flags.go b/internal/metamorphic/metaflags/meta_flags.go
index b8b5ed84..049a701a 100644
--- a/internal/metamorphic/metaflags/meta_flags.go
+++ b/internal/metamorphic/metaflags/meta_flags.go
@@ -60,7 +60,8 @@ func (c *CommonFlags) KeyFormat() metamorphic.KeyFormat {
 
 // KeyFormats is a map of available key formats.
 var KeyFormats = map[string]metamorphic.KeyFormat{
-	"testkeys": metamorphic.TestkeysKeyFormat,
+	"testkeys":     metamorphic.TestkeysKeyFormat,
+	"cockroachkvs": metamorphic.CockroachKeyFormat,
 }
 
 func initCommonFlags() *CommonFlags {
diff --git a/internal/rawalloc/rawalloc_32bit.go b/internal/rawalloc/rawalloc_32bit.go
index 3112cc9e..1046d397 100644
--- a/internal/rawalloc/rawalloc_32bit.go
+++ b/internal/rawalloc/rawalloc_32bit.go
@@ -13,7 +13,6 @@
 // permissions and limitations under the License.
 
 //go:build 386 || amd64p32 || arm || armbe || ppc || sparc
-// +build 386 amd64p32 arm armbe ppc sparc
 
 package rawalloc
 
diff --git a/internal/rawalloc/rawalloc_64bit.go b/internal/rawalloc/rawalloc_64bit.go
index 6660462e..f07420dc 100644
--- a/internal/rawalloc/rawalloc_64bit.go
+++ b/internal/rawalloc/rawalloc_64bit.go
@@ -13,7 +13,6 @@
 // permissions and limitations under the License.
 
 //go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64 || riscv64 || loong64
-// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le s390x sparc64 riscv64 loong64
 
 package rawalloc
 
diff --git a/internal/rawalloc/rawalloc_gccgo.go b/internal/rawalloc/rawalloc_gccgo.go
index f2db79c7..716fe6c8 100644
--- a/internal/rawalloc/rawalloc_gccgo.go
+++ b/internal/rawalloc/rawalloc_gccgo.go
@@ -13,7 +13,6 @@
 // permissions and limitations under the License.
 
 //go:build gccgo
-// +build gccgo
 
 package rawalloc
 
diff --git a/internal/rawalloc/rawalloc_go1.9.go b/internal/rawalloc/rawalloc_go1.9.go
index 65da4361..055f6714 100644
--- a/internal/rawalloc/rawalloc_go1.9.go
+++ b/internal/rawalloc/rawalloc_go1.9.go
@@ -13,7 +13,6 @@
 // permissions and limitations under the License.
 
 //go:build gc && go1.9
-// +build gc,go1.9
 
 package rawalloc
 
diff --git a/internal/rawalloc/rawalloc_mipsall.go b/internal/rawalloc/rawalloc_mipsall.go
index 55b45eb1..badb0367 100644
--- a/internal/rawalloc/rawalloc_mipsall.go
+++ b/internal/rawalloc/rawalloc_mipsall.go
@@ -13,7 +13,6 @@
 // permissions and limitations under the License.
 
 //go:build mips || mipsle || mips64p32 || mips64p32le
-// +build mips mipsle mips64p32 mips64p32le
 
 package rawalloc
 
diff --git a/internal/sstableinternal/options.go b/internal/sstableinternal/options.go
index 3dccebdb..2fd056e2 100644
--- a/internal/sstableinternal/options.go
+++ b/internal/sstableinternal/options.go
@@ -12,11 +12,10 @@ import (
 // CacheOptions contains the information needed to interact with the block
 // cache.
 type CacheOptions struct {
-	// Cache can be nil, in which case no cache is used. When non-nil, the other
+	// CacheHandle can be nil, in which case no cache is used. When non-nil, the other
 	// fields must be set accordingly.
-	Cache   *cache.Cache
-	CacheID cache.ID
-	FileNum base.DiskFileNum
+	CacheHandle *cache.Handle
+	FileNum     base.DiskFileNum
 }
 
 // ReaderOptions are fields of sstable.ReaderOptions that can only be set from
diff --git a/iterator.go b/iterator.go
index 84edd16c..230fb2c2 100644
--- a/iterator.go
+++ b/iterator.go
@@ -22,7 +22,6 @@ import (
 	"github.com/cockroachdb/pebble/internal/manifest"
 	"github.com/cockroachdb/pebble/internal/rangekeystack"
 	"github.com/cockroachdb/pebble/internal/treeprinter"
-	"github.com/cockroachdb/pebble/sstable"
 	"github.com/cockroachdb/redact"
 )
 
@@ -239,13 +238,12 @@ type Iterator struct {
 	prefixOrFullSeekKey []byte
 	readSampling        readSampling
 	stats               IteratorStats
-	externalReaders     [][]*sstable.Reader
-
+	externalIter        *externalIterState
 	// Following fields used when constructing an iterator stack, eg, in Clone
 	// and SetOptions or when re-fragmenting a batch's range keys/range dels.
 	// Non-nil if this Iterator includes a Batch.
 	batch            *Batch
-	fc               *fileCacheContainer
+	fc               *fileCacheHandle
 	newIters         tableNewIters
 	newIterRangeKey  keyspanimpl.TableNewSpanIter
 	lazyCombinedIter lazyCombinedIter
@@ -2387,11 +2385,8 @@ func (i *Iterator) Close() error {
 	if i.version != nil {
 		i.version.Unref()
 	}
-
-	for _, readers := range i.externalReaders {
-		for _, r := range readers {
-			err = firstError(err, r.Close())
-		}
+	if i.externalIter != nil {
+		err = firstError(err, i.externalIter.Close())
 	}
 
 	// Close the closer for the current value if one was open.
@@ -2401,7 +2396,6 @@ func (i *Iterator) Close() error {
 	}
 
 	if i.rangeKey != nil {
-
 		i.rangeKey.rangeKeyBuffers.PrepareForReuse()
 		*i.rangeKey = iteratorRangeKeyState{
 			rangeKeyBuffers: i.rangeKey.rangeKeyBuffers,
@@ -2581,7 +2575,7 @@ func (i *Iterator) processBounds(lower, upper []byte) {
 //
 // If only lower and upper bounds need to be modified, prefer SetBounds.
 func (i *Iterator) SetOptions(o *IterOptions) {
-	if i.externalReaders != nil {
+	if i.externalIter != nil {
 		if err := validateExternalIterOpts(o); err != nil {
 			panic(err)
 		}
@@ -2753,7 +2747,7 @@ func (i *Iterator) SetOptions(o *IterOptions) {
 
 	// Iterators created through NewExternalIter have a different iterator
 	// initialization process.
-	if i.externalReaders != nil {
+	if i.externalIter != nil {
 		finishInitializingExternal(i.ctx, i)
 		return
 	}
diff --git a/level_checker.go b/level_checker.go
index 49d518af..9ed051d8 100644
--- a/level_checker.go
+++ b/level_checker.go
@@ -314,7 +314,7 @@ type tombstoneWithLevel struct {
 	level int
 	// The level in LSM. A -1 means it's a memtable.
 	lsmLevel int
-	fileNum  FileNum
+	fileNum  base.FileNum
 }
 
 // For sorting tombstoneWithLevels in increasing order of start UserKey and
@@ -443,7 +443,7 @@ func checkRangeTombstones(c *checkConfig) error {
 	return iterateAndCheckTombstones(c.cmp, c.formatKey, tombstones)
 }
 
-func levelOrMemtable(lsmLevel int, fileNum FileNum) string {
+func levelOrMemtable(lsmLevel int, fileNum base.FileNum) string {
 	if lsmLevel == -1 {
 		return "memtable"
 	}
@@ -454,7 +454,7 @@ func addTombstonesFromIter(
 	iter keyspan.FragmentIterator,
 	level int,
 	lsmLevel int,
-	fileNum FileNum,
+	fileNum base.FileNum,
 	tombstones []tombstoneWithLevel,
 	seqNum base.SeqNum,
 	cmp Compare,
diff --git a/level_iter.go b/level_iter.go
index 11d39d37..9e8b08c3 100644
--- a/level_iter.go
+++ b/level_iter.go
@@ -26,7 +26,7 @@ type internalIterOpts struct {
 	compaction           bool
 	bufferPool           *block.BufferPool
 	stats                *base.InternalIteratorStats
-	iterStatsAccumulator block.IterStatsAccumulator
+	iterStatsAccumulator *block.CategoryStatsShard
 	boundLimitedFilter   sstable.BoundLimitedBlockPropertyFilter
 }
 
diff --git a/mem_table.go b/mem_table.go
index d31799f5..9f942ade 100644
--- a/mem_table.go
+++ b/mem_table.go
@@ -119,7 +119,10 @@ func checkMemTable(obj interface{}) {
 // newMemTable returns a new MemTable of the specified size. If size is zero,
 // Options.MemTableSize is used instead.
 func newMemTable(opts memTableOptions) *memTable {
-	opts.Options = opts.Options.EnsureDefaults()
+	if opts.Options == nil {
+		opts.Options = &Options{}
+	}
+	opts.Options.EnsureDefaults()
 	m := new(memTable)
 	m.init(opts)
 	return m
diff --git a/metamorphic/build.go b/metamorphic/build.go
index 4b5af83a..8c8bee8c 100644
--- a/metamorphic/build.go
+++ b/metamorphic/build.go
@@ -97,6 +97,7 @@ func writeSSTForIngestion(
 		if err != nil {
 			return nil, err
 		}
+		t.opts.Comparer.ValidateKey.MustValidate(k.K.UserKey)
 		if err := w.Raw().AddWithForceObsolete(k.K, valBytes, false); err != nil {
 			return nil, err
 		}
@@ -111,7 +112,11 @@ func writeSSTForIngestion(
 			if syntheticSuffix.IsSet() {
 				panic("synthetic suffix with RangeDel")
 			}
-			if err := w.DeleteRange(outputKey(span.Start, nil), outputKey(span.End, nil)); err != nil {
+			start := outputKey(span.Start, nil)
+			end := outputKey(span.End, nil)
+			t.opts.Comparer.ValidateKey.MustValidate(start)
+			t.opts.Comparer.ValidateKey.MustValidate(end)
+			if err := w.DeleteRange(start, end); err != nil {
 				return nil, err
 			}
 		}
@@ -139,6 +144,8 @@ func writeSSTForIngestion(
 				End:   outputKey(span.End, nil),
 				Keys:  make([]keyspan.Key, 0, len(span.Keys)),
 			}
+			t.opts.Comparer.ValidateKey.MustValidate(collapsed.Start)
+			t.opts.Comparer.ValidateKey.MustValidate(collapsed.End)
 			keys := span.Keys
 			if syntheticSuffix.IsSet() {
 				keys = slices.Clone(span.Keys)
diff --git a/metamorphic/cockroachkvs.go b/metamorphic/cockroachkvs.go
new file mode 100644
index 00000000..77b096a1
--- /dev/null
+++ b/metamorphic/cockroachkvs.go
@@ -0,0 +1,422 @@
+// Copyright 2025 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package metamorphic
+
+import (
+	"fmt"
+	"math/rand/v2"
+	"slices"
+
+	"github.com/cockroachdb/pebble"
+	"github.com/cockroachdb/pebble/cockroachkvs"
+	"github.com/cockroachdb/pebble/internal/testkeys"
+	"github.com/cockroachdb/pebble/sstable"
+	"github.com/cockroachdb/pebble/sstable/colblk"
+)
+
+// CockroachKeyFormat provides a KeyFormat implementation that uses
+// CockroachDB's key encoding (as defined in the cockroachkvs package).
+var CockroachKeyFormat = KeyFormat{
+	Name:                    "cockroachkvs",
+	Comparer:                &cockroachkvs.Comparer,
+	KeySchema:               func() *colblk.KeySchema { return &cockroachkvs.KeySchema }(),
+	BlockPropertyCollectors: cockroachkvs.BlockPropertyCollectors,
+	FormatKey: func(k UserKey) string {
+		if len(k) == 0 {
+			return ""
+		}
+		return fmt.Sprint(cockroachkvs.FormatKey(k))
+	},
+	FormatKeySuffix: func(s UserKeySuffix) string {
+		if len(s) == 0 {
+			return ""
+		}
+		return fmt.Sprint(cockroachkvs.FormatKeySuffix(s))
+	},
+	ParseFormattedKey: func(formattedKey string) UserKey {
+		return UserKey(cockroachkvs.ParseFormattedKey(formattedKey))
+	},
+	ParseFormattedKeySuffix: func(formattedKeySuffix string) UserKeySuffix {
+		return UserKeySuffix(cockroachkvs.ParseFormattedKeySuffix(formattedKeySuffix))
+	},
+	NewGenerator: func(km *keyManager, rng *rand.Rand, cfg OpConfig) KeyGenerator {
+		return &cockroachKeyGenerator{
+			keyManager: km,
+			rng:        rng,
+			cfg:        cfg,
+			// TODO(jackson): Vary maxLogical.
+			suffixSpace: cockroachSuffixKeyspace{maxLogical: 2},
+		}
+	},
+	NewSuffixFilterMask: func() pebble.BlockPropertyFilterMask {
+		return &cockroachkvs.MVCCWallTimeIntervalRangeKeyMask{}
+	},
+	NewSuffixBlockPropertyFilter: func(minSuffix, maxSuffix []byte) sstable.BlockPropertyFilter {
+		minWallTime, _, err := cockroachkvs.DecodeMVCCTimestampSuffix(maxSuffix)
+		if err != nil {
+			panic(err)
+		}
+		maxWallTime, _, err := cockroachkvs.DecodeMVCCTimestampSuffix(minSuffix)
+		if err != nil {
+			panic(err)
+		}
+		return cockroachkvs.NewMVCCTimeIntervalFilter(minWallTime, maxWallTime)
+	},
+}
+
+type cockroachKeyGenerator struct {
+	keyManager  *keyManager
+	rng         *rand.Rand
+	cfg         OpConfig
+	suffixSpace cockroachSuffixKeyspace
+}
+
+// RecordPrecedingKey may be invoked before generating keys to inform the key
+// generator of a key that was previously generated and used within a related
+// test context.
+func (kg *cockroachKeyGenerator) RecordPrecedingKey(key []byte) {
+	// If the key has a suffix that's larger than the current max suffix,
+	// ratchet up the maximum of the distribution of suffixes.
+	if i := cockroachkvs.Comparer.Split(key); i < len(key) {
+		suffixIdx := kg.suffixSpace.ToSuffixIndex(key[i:])
+		if suffixIdx > suffixIndex(kg.cfg.writeSuffixDist.Max()) {
+			diff := uint64(suffixIdx) - kg.cfg.writeSuffixDist.Max()
+			kg.cfg.writeSuffixDist.IncMax(diff)
+		}
+	}
+}
+
+// ExtendPrefix extends the given prefix key with additional bytes,
+// returning a new prefix that sorts after the given prefix.
+func (kg *cockroachKeyGenerator) ExtendPrefix(prefix []byte) []byte {
+	// Copy prefix and strip the delimiter byte.
+	p := append(make([]byte, 0, len(prefix)+3), prefix[:len(prefix)-1]...)
+	p = append(p, randBytes(kg.rng, 1, 3)...)
+	p = append(p, 0x00) // Delimiter byte
+	return p
+}
+
+// RandKey returns a random key (either a previously known key, or a new key).
+func (kg *cockroachKeyGenerator) RandKey(newKeyProbability float64) []byte {
+	return kg.randKey(newKeyProbability, nil /* bounds */)
+}
+
+// RandKeyInRange returns a random key (either a previously known key, or a new
+// key) in the given key range.
+func (kg *cockroachKeyGenerator) RandKeyInRange(
+	newKeyProbability float64, kr pebble.KeyRange,
+) []byte {
+	return kg.randKey(newKeyProbability, &kr)
+}
+
+// RandPrefix returns a random prefix key (a key with no suffix).
+func (kg *cockroachKeyGenerator) RandPrefix(newPrefix float64) []byte {
+	prefixes := kg.keyManager.prefixes()
+	if len(prefixes) > 0 && kg.rng.Float64() > newPrefix {
+		return pickOneUniform(kg.rng, prefixes)
+	}
+
+	// Use a new prefix.
+	for {
+		prefix := kg.generateKeyWithSuffix(4, 12, 0)
+		if !kg.keyManager.prefixExists(prefix) {
+			if !kg.keyManager.addNewKey(prefix) {
+				panic("key must not exist if prefix doesn't exist")
+			}
+			return prefix
+		}
+	}
+}
+
+// SkewedSuffix generates a random suffix according to the configuration's
+// suffix distribution. It takes a probability 0 ≤ p ≤ 1.0 indicating the
+// probability with which the generator should increase the max suffix generated
+// by the generator.
+//
+// May return a nil suffix, with the probability the configuration's suffix
+// distribution assigns to the zero suffix.
+func (kg *cockroachKeyGenerator) SkewedSuffix(incMaxProb float64) []byte {
+	if suffixIdx := kg.skewedSuffixInt(incMaxProb); suffixIdx != 0 {
+		return kg.suffixSpace.ToMaterializedSuffix(suffixIdx)
+	}
+	return nil
+}
+
+// skewedSuffixInt is a helper of SkewedSuffix which returns the unencoded
+// suffix as an integer.
+func (kg *cockroachKeyGenerator) skewedSuffixInt(incMaxProb float64) suffixIndex {
+	if kg.rng.Float64() < incMaxProb {
+		kg.cfg.writeSuffixDist.IncMax(1)
+	}
+	return suffixIndex(kg.cfg.writeSuffixDist.Uint64(kg.rng))
+}
+
+// IncMaxSuffix increases the max suffix range and returns the new maximum
+// suffix (which is guaranteed to be larger than any previously generated
+// suffix).
+func (kg *cockroachKeyGenerator) IncMaxSuffix() []byte {
+	kg.cfg.writeSuffixDist.IncMax(1)
+	s := suffixIndex(kg.cfg.writeSuffixDist.Max())
+	return kg.suffixSpace.ToMaterializedSuffix(s)
+}
+
+// SuffixRange generates a new uniformly random range of suffixes (low, high]
+// such that high is guaranteed to be strictly greater (as defined by
+// ComparePointSuffixes) than low.
+//
+// The high suffix may be nil, in which case the suffix range represents all
+// suffixes ≥ low.
+func (kg *cockroachKeyGenerator) SuffixRange() (low, high []byte) {
+	a := kg.uniformSuffixInt()
+	b := kg.uniformSuffixInt()
+	if a < b {
+		a, b = b, a
+	} else if a == b {
+		a++
+	}
+	return kg.suffixSpace.ToMaterializedSuffix(a), kg.suffixSpace.ToMaterializedSuffix(b)
+}
+
+// UniformSuffix returns a suffix in the same range as SkewedSuffix but with a
+// uniform distribution. This is used during reads to better exercise reading a
+// mix of older and newer keys. The suffix can be empty.
+//
+// May return a nil suffix.
+func (kg *cockroachKeyGenerator) UniformSuffix() []byte {
+	if suffix := kg.uniformSuffixInt(); suffix != 0 {
+		return kg.suffixSpace.ToMaterializedSuffix(suffix)
+	}
+	return nil
+}
+
+// uniformSuffixInt is a helper of UniformSuffix which returns the suffix
+// index.
+func (kg *cockroachKeyGenerator) uniformSuffixInt() suffixIndex {
+	maxVal := kg.cfg.writeSuffixDist.Max()
+	return suffixIndex(kg.rng.Int64N(int64(maxVal)))
+}
+
+// randKey returns a random key (either a previously known key or a new key).
+//
+// If bounds is not nil, the key will be inside the bounds.
+func (kg *cockroachKeyGenerator) randKey(
+	newKeyProbability float64, bounds *pebble.KeyRange,
+) []byte {
+	var knownKeys [][]byte
+	if bounds == nil {
+		knownKeys = kg.keyManager.knownKeys()
+	} else {
+		if cockroachkvs.Compare(bounds.Start, bounds.End) >= 0 {
+			panic(fmt.Sprintf("invalid bounds [%q, %q)", bounds.Start, bounds.End))
+		}
+		knownKeys = kg.keyManager.knownKeysInRange(*bounds)
+	}
+	switch {
+	case len(knownKeys) > 0 && kg.rng.Float64() > newKeyProbability:
+		// Use an existing user key.
+		return pickOneUniform(kg.rng, knownKeys)
+
+	case len(knownKeys) > 0 && kg.rng.Float64() > kg.cfg.newPrefix:
+		// Use an existing prefix but a new suffix, producing a new user key.
+		prefixes := kg.keyManager.prefixes()
+
+		// If we're constrained to a key range, find which existing prefixes
+		// fall within that key range.
+		if bounds != nil {
+			s, _ := slices.BinarySearchFunc(prefixes, bounds.Start, cockroachkvs.Compare)
+			e, _ := slices.BinarySearchFunc(prefixes, bounds.End, cockroachkvs.Compare)
+			prefixes = prefixes[s:e]
+		}
+
+		if len(prefixes) > 0 {
+			for {
+				// Pick a prefix on each iteration in case most or all suffixes are
+				// already in use for any individual prefix.
+				p := kg.rng.IntN(len(prefixes))
+				suffix := suffixIndex(kg.cfg.writeSuffixDist.Uint64(kg.rng))
+
+				var key []byte
+				if suffix > 0 {
+					key = append(append(key, prefixes[p]...), kg.suffixSpace.ToMaterializedSuffix(suffix)...)
+				} else {
+					key = append(key, prefixes[p]...)
+				}
+				if bounds == nil || (cockroachkvs.Compare(key, bounds.Start) >= 0 &&
+					cockroachkvs.Compare(key, bounds.End) < 0) {
+					if kg.keyManager.addNewKey(key) {
+						return key
+					}
+				}
+
+				// If the generated key already existed, or the generated key
+				// fell outside the provided bounds, increase the suffix
+				// distribution and loop.
+				kg.cfg.writeSuffixDist.IncMax(1)
+			}
+		}
+		// Otherwise fall through to generating a new prefix.
+	}
+
+	if bounds == nil {
+		suffixIdx := kg.skewedSuffixInt(0.01)
+		for {
+			key := kg.generateKeyWithSuffix(4, 12, suffixIdx)
+			if !kg.keyManager.prefixExists(kg.keyManager.kf.Comparer.Split.Prefix(key)) {
+				if !kg.keyManager.addNewKey(key) {
+					panic("key must not exist if prefix doesn't exist")
+				}
+				return key
+			}
+		}
+	}
+	// We need to generate a key between the bounds.
+	startPrefix, startSuffixIdx := kg.suffixSpace.Split(bounds.Start)
+	endPrefix, endSuffixIdx := kg.suffixSpace.Split(bounds.End)
+
+	var prefix []byte
+	var suffixIdx suffixIndex
+	if cockroachkvs.Equal(startPrefix, endPrefix) {
+		prefix = startPrefix
+		// Bounds have the same prefix, generate a suffix in-between.
+		if startSuffixIdx <= endSuffixIdx {
+			panic(fmt.Sprintf("invalid bounds [%q, %q)", bounds.Start, bounds.End))
+		}
+		suffixIdx = kg.skewedSuffixInt(0.01)
+		for i := 0; !(startSuffixIdx >= suffixIdx && endSuffixIdx < suffixIdx); i++ {
+			if i > 10 {
+				// This value is always >= startSuffix and < endSuffix.
+				suffixIdx = (startSuffixIdx + endSuffixIdx) / 2
+				break
+			}
+			// The suffix we want must exist in the current suffix range, we don't
+			// want to keep increasing it here.
+			suffixIdx = kg.skewedSuffixInt(0)
+		}
+	} else {
+		prefix = append(testkeys.RandomPrefixInRange(
+			startPrefix[:len(startPrefix)-1], // Strip the delimiter byte.
+			endPrefix[:len(endPrefix)-1],     // Strip the delimiter byte.
+			kg.rng,
+		), 0x00) // Add back the delimiter byte.
+		suffixIdx = kg.skewedSuffixInt(0.01)
+		if cockroachkvs.Equal(prefix, startPrefix) {
+			// We can't use a suffix which sorts before startSuffix.
+			for i := 0; suffixIdx > startSuffixIdx; i++ {
+				if i > 10 {
+					suffixIdx = startSuffixIdx
+					break
+				}
+				suffixIdx = kg.skewedSuffixInt(0)
+			}
+		}
+	}
+	key := slices.Clip(prefix)
+	if suffixIdx != 0 {
+		key = append(key, kg.suffixSpace.ToMaterializedSuffix(suffixIdx)...)
+	}
+	if cockroachkvs.Compare(key, bounds.Start) < 0 || cockroachkvs.Compare(key, bounds.End) >= 0 {
+		panic(fmt.Sprintf("invalid randKey %q; bounds: [%q, %q) %v %v",
+			key, bounds.Start, bounds.End,
+			cockroachkvs.Compare(key, bounds.Start),
+			cockroachkvs.Compare(key, bounds.End)))
+	}
+	// We might (rarely) produce an existing key here, that's ok.
+	kg.keyManager.addNewKey(key)
+	return key
+}
+
+// generateKeyWithSuffix generates a key with a random prefix and the suffix
+// corresponding to the provided suffix index. If the given suffix index is 0,
+// the key will not have a suffix.
+func (kg *cockroachKeyGenerator) generateKeyWithSuffix(
+	minPrefixLen, maxPrefixLen int, suffixIdx suffixIndex,
+) []byte {
+	prefix := randCockroachPrefix(kg.rng, minPrefixLen, maxPrefixLen)
+	if suffixIdx == 0 {
+		return prefix
+	}
+	return append(prefix, kg.suffixSpace.ToMaterializedSuffix(suffixIdx)...)
+}
+
+func randCockroachPrefix(rng *rand.Rand, minLen, maxLen int) []byte {
+	n := minLen + rng.IntN(maxLen-minLen+1)
+	if n == 0 {
+		return nil
+	}
+	// NB: The actual random values are not particularly important. We only use
+	// lowercase letters because that makes visual determination of ordering
+	// easier, rather than having to remember the lexicographic ordering of
+	// uppercase vs lowercase, or letters vs numbers vs punctuation.
+	const letters = "abcdefghijklmnopqrstuvwxyz"
+	const lettersLen = uint64(len(letters))
+	const lettersCharsPerRand = 12 // floor(log(math.MaxUint64)/log(lettersLen))
+
+	var r uint64
+	var q int
+	buf := make([]byte, n+1)
+	for i := 0; i < n; i++ {
+		if q == 0 {
+			r = rng.Uint64()
+			q = lettersCharsPerRand
+		}
+		buf[i] = letters[r%lettersLen]
+		r = r / lettersLen
+		q--
+	}
+	buf[n] = 0x00 // Delimiter byte
+	return buf
+}
+
+// A suffixIndex represents a unique suffix. The suffixIndex exists within a
+// one-dimensional space of int64s, but is remapped into a two-dimensional space
+// of MVCC timestamps of (WallTime, Logical) tuples.
+type suffixIndex int64
+
+// cockroackSuffixKeyspace defines the mapping between a one-dimensional
+// suffixIndex and the suffix it represents within the two-dimensional
+// (wallTime, logical) space.
+//
+// The mapping is configued by the value of maxLogical, with maxLogical+1
+// possible logical timestamps at each wall time.
+//
+// TODO(jackson): Update to disallow non-zero logical timestamps when the wall
+// time is zero if we begin to prohibit it.
+//
+//	+------------------------------------------------------+
+//	| suffix  |              maxLogical                    |
+//	| index   | 0      | 1      | 2      | 3      | 4      |
+//	+------------------------------------------------------+
+//	|  0      | (0,0)  | (0,0)  | (0,0)  | (0,0)  | (0,0)  |
+//	|  1      | (1,0)  | (0,1)  | (0,1)  | (0,1)  | (0,1)  |
+//	|  2      | (2,0)  | (1,0)  | (0,2)  | (0,2)  | (0,2)  |
+//	|  3      | (3,0)  | (1,1)  | (1,0)  | (0,3)  | (0,3)  |
+//	|  4      | (4,0)  | (2,0)  | (1,1)  | (1,0)  | (0,4)  |
+//	|  5      | (5,0)  | (2,1)  | (1,2)  | (1,1)  | (1,0)  |
+//	|  6      | (6,0)  | (3,0)  | (2,0)  | (1,2)  | (1,1)  |
+//	|  7      | (7,0)  | (3,1)  | (2,1)  | (1,3)  | (1,2)  |
+//	+------------------------------------------------------+
+type cockroachSuffixKeyspace struct {
+	maxLogical int64
+}
+
+func (ks cockroachSuffixKeyspace) ToMaterializedSuffix(s suffixIndex) []byte {
+	// There are maxLogical+1 possible logical timestamps at each wall time.
+	wallTime := int64(s) / (ks.maxLogical + 1)
+	logical := int64(s) % (ks.maxLogical + 1)
+	return cockroachkvs.NewTimestampSuffix(uint64(wallTime), uint32(logical))
+}
+
+func (ks cockroachSuffixKeyspace) ToSuffixIndex(suffix []byte) suffixIndex {
+	wallTime, logical, err := cockroachkvs.DecodeMVCCTimestampSuffix(suffix)
+	if err != nil {
+		panic(err)
+	}
+	return suffixIndex(int64(wallTime)*(ks.maxLogical+1) + int64(logical))
+}
+
+func (ks cockroachSuffixKeyspace) Split(key []byte) (prefix []byte, suffixIdx suffixIndex) {
+	i := cockroachkvs.Split(key)
+	return key[:i], ks.ToSuffixIndex(key[i:])
+}
diff --git a/metamorphic/config.go b/metamorphic/config.go
index 3efd2ae2..f12be213 100644
--- a/metamorphic/config.go
+++ b/metamorphic/config.go
@@ -345,6 +345,7 @@ func mustDynamic(dyn randvar.Dynamic, err error) randvar.Dynamic {
 
 var knownKeyFormats = []KeyFormat{
 	TestkeysKeyFormat,
+	CockroachKeyFormat,
 }
 
 var keyFormatsByName = func() map[string]KeyFormat {
@@ -422,8 +423,12 @@ type KeyGenerator interface {
 	// May return a nil suffix, with the probability the configuration's suffix
 	// distribution assigns to the zero suffix.
 	SkewedSuffix(incMaxProb float64) []byte
-	// SuffixRange generates a new uniformly random range of suffixes [low,
-	// high) such that high is guaranteed to be strictly greater than low.
+	// SuffixRange generates a new uniformly random range of suffixes (low, high]
+	// such that high is guaranteed to be strictly greater (as defined by
+	// ComparePointSuffixes) than low.
+	//
+	// The high suffix may be nil, in which case the suffix range represents all
+	// suffixes ≥ low.
 	SuffixRange() (low, high []byte)
 	// UniformSuffix returns a suffix in the same range as SkewedSuffix but with
 	// a uniform distribution. This is used during reads to better exercise
diff --git a/metamorphic/generator.go b/metamorphic/generator.go
index 2ceb68e0..257c420b 100644
--- a/metamorphic/generator.go
+++ b/metamorphic/generator.go
@@ -29,7 +29,13 @@ type iterOpts struct {
 	maskSuffix UserKeySuffix
 
 	// If filterMax is != nil, this iterator will filter out any keys that have
-	// suffixes that don't fall within the range [filterMin,filterMax).
+	// suffixes that don't fall within the range (filterMin,filterMax]
+	// [according to the ordering defined by ComparePointSuffixes]. Note that
+	// suffixes are used to represent MVCC timestamps, and MVCC timestamps are
+	// ordered in numerically descending order, so the timestamp associated with
+	// filterMin is more recent than that associated with filterMax. This means
+	// according to the ordering of ComparePointSuffixes, filterMin > filterMax.
+	//
 	// Additionally, the iterator will be constructed with a block-property
 	// filter that filters out blocks accordingly. Not all OPTIONS hook up the
 	// corresponding block property collector, so block-filtering may still be
@@ -1529,7 +1535,7 @@ func (g *generator) maybeMutateOptions(readerID objID, opts *iterOpts) {
 		}
 
 		// With 1/3 probability, clear existing filter.
-		if opts.filterMax != nil && g.rng.IntN(3) == 0 {
+		if opts.filterMin != nil && g.rng.IntN(3) == 0 {
 			opts.filterMax, opts.filterMin = nil, nil
 		}
 		// With 10% probability, set a filter range.
diff --git a/metamorphic/key_manager.go b/metamorphic/key_manager.go
index a3622be5..5c1cbd06 100644
--- a/metamorphic/key_manager.go
+++ b/metamorphic/key_manager.go
@@ -389,6 +389,7 @@ func newKeyManager(numInstances int, kf KeyFormat) *keyManager {
 // addNewKey adds the given key to the key manager for global key tracking.
 // Returns false iff this is not a new key.
 func (k *keyManager) addNewKey(key []byte) bool {
+	k.kf.Comparer.ValidateKey.MustValidate(key)
 	if k.globalKeysMap[string(key)] {
 		return false
 	}
@@ -409,6 +410,7 @@ func (k *keyManager) addNewKey(key []byte) bool {
 // getOrInit returns the keyMeta for the (objID, key) pair, if it exists, else
 // allocates, initializes and returns a new value.
 func (k *keyManager) getOrInit(id objID, key []byte) *keyMeta {
+	k.kf.Comparer.ValidateKey.MustValidate(key)
 	objKeys := k.objKeyMeta(id)
 	m, ok := objKeys.keys[string(key)]
 	if ok {
diff --git a/metamorphic/ops.go b/metamorphic/ops.go
index e1dae3bf..5b59e186 100644
--- a/metamorphic/ops.go
+++ b/metamorphic/ops.go
@@ -811,6 +811,7 @@ func (o *ingestOp) collapseBatch(
 	if pointIter != nil {
 		var lastUserKey []byte
 		for kv := pointIter.First(); kv != nil; kv = pointIter.Next() {
+			t.opts.Comparer.ValidateKey.MustValidate(kv.K.UserKey)
 			// Ignore duplicate keys.
 			//
 			// Note: this is necessary due to MERGE keys, otherwise it would be
@@ -882,6 +883,7 @@ func (o *ingestOp) collapseBatch(
 				continue
 			}
 			ik := base.MakeInternalKey(key, 0, kind)
+			t.opts.Comparer.ValidateKey.MustValidate(ik.UserKey)
 			if err := collapsed.AddInternalKey(&ik, value, nil); err != nil {
 				return nil, err
 			}
@@ -943,6 +945,8 @@ func (o *ingestAndExciseOp) run(t *Test, h historyRecorder) {
 		return
 	}
 
+	t.opts.Comparer.ValidateKey.MustValidate(o.exciseStart)
+	t.opts.Comparer.ValidateKey.MustValidate(o.exciseEnd)
 	if t.testOpts.useExcise {
 		err = firstError(err, t.withRetries(func() error {
 			_, err := db.IngestAndExcise(context.Background(), []string{path}, nil /* shared */, nil /* external */, pebble.KeyRange{
@@ -1043,6 +1047,9 @@ func (o *ingestExternalFilesOp) run(t *Test, h historyRecorder) {
 	} else {
 		external := make([]pebble.ExternalFile, len(o.objs))
 		for i, obj := range o.objs {
+			t.opts.Comparer.ValidateKey.MustValidate(obj.bounds.Start)
+			t.opts.Comparer.ValidateKey.MustValidate(obj.bounds.End)
+
 			meta := t.getExternalObj(obj.externalObjID)
 			external[i] = pebble.ExternalFile{
 				Locator:           "external",
@@ -1201,7 +1208,7 @@ func (o *newIterOp) run(t *Test, h historyRecorder) {
 func (o *newIterOp) formattedString(kf KeyFormat) string {
 	return fmt.Sprintf("%s = %s.NewIter(%q, %q, %d /* key types */, %q, %q, %t /* use L6 filters */, %q /* masking suffix */)",
 		o.iterID, o.readerID, kf.FormatKey(o.lower), kf.FormatKey(o.upper),
-		o.keyTypes, kf.FormatKeySuffix(o.filterMin), kf.FormatKeySuffix(o.filterMax),
+		o.keyTypes, kf.FormatKeySuffix(o.filterMax), kf.FormatKeySuffix(o.filterMin),
 		o.useL6Filters, kf.FormatKeySuffix(o.maskSuffix))
 }
 
@@ -1268,8 +1275,8 @@ func (o *newIterUsingCloneOp) run(t *Test, h historyRecorder) {
 func (o *newIterUsingCloneOp) formattedString(kf KeyFormat) string {
 	return fmt.Sprintf("%s = %s.Clone(%t, %q, %q, %d /* key types */, %q, %q, %t /* use L6 filters */, %q /* masking suffix */)",
 		o.iterID, o.existingIterID, o.refreshBatch, kf.FormatKey(o.lower),
-		kf.FormatKey(o.upper), o.keyTypes, kf.FormatKeySuffix(o.filterMin),
-		kf.FormatKeySuffix(o.filterMax), o.useL6Filters, kf.FormatKeySuffix(o.maskSuffix))
+		kf.FormatKey(o.upper), o.keyTypes, kf.FormatKeySuffix(o.filterMax),
+		kf.FormatKeySuffix(o.filterMin), o.useL6Filters, kf.FormatKeySuffix(o.maskSuffix))
 }
 
 func (o *newIterUsingCloneOp) receiver() objID { return o.existingIterID }
@@ -1365,7 +1372,7 @@ func (o *iterSetOptionsOp) run(t *Test, h historyRecorder) {
 func (o *iterSetOptionsOp) formattedString(kf KeyFormat) string {
 	return fmt.Sprintf("%s.SetOptions(%q, %q, %d /* key types */, %q, %q, %t /* use L6 filters */, %q /* masking suffix */)",
 		o.iterID, kf.FormatKey(o.lower), kf.FormatKey(o.upper),
-		o.keyTypes, kf.FormatKeySuffix(o.filterMin), kf.FormatKeySuffix(o.filterMax),
+		o.keyTypes, kf.FormatKeySuffix(o.filterMax), kf.FormatKeySuffix(o.filterMin),
 		o.useL6Filters, kf.FormatKeySuffix(o.maskSuffix))
 }
 
@@ -1393,10 +1400,13 @@ func iterOptions(kf KeyFormat, o iterOpts) *pebble.IterOptions {
 	if opts.RangeKeyMasking.Suffix != nil {
 		opts.RangeKeyMasking.Filter = kf.NewSuffixFilterMask
 	}
-	if o.filterMax != nil {
+	if o.filterMin != nil {
 		opts.PointKeyFilters = []pebble.BlockPropertyFilter{
 			kf.NewSuffixBlockPropertyFilter(o.filterMin, o.filterMax),
 		}
+		if o.filterMax != nil && kf.Comparer.ComparePointSuffixes(o.filterMin, o.filterMax) >= 0 {
+			panic(errors.AssertionFailedf("filterMin >= filterMax: %q >= %q", o.filterMin, o.filterMax))
+		}
 		// Enforce the timestamp bounds in SkipPoint, so that the iterator never
 		// returns a key outside the filterMin, filterMax bounds. This provides
 		// deterministic iteration.
@@ -1410,10 +1420,10 @@ func iterOptions(kf KeyFormat, o iterOpts) *pebble.IterOptions {
 				// non-timestamped keys.
 				return true
 			}
-			if kf.Comparer.ComparePointSuffixes(k[n:], o.filterMin) < 0 {
+			if kf.Comparer.ComparePointSuffixes(k[n:], o.filterMin) <= 0 {
 				return true
 			}
-			if kf.Comparer.ComparePointSuffixes(k[n:], o.filterMax) >= 0 {
+			if o.filterMax != nil && kf.Comparer.ComparePointSuffixes(k[n:], o.filterMax) > 0 {
 				return true
 			}
 			return false
@@ -1446,9 +1456,9 @@ type iterSeekGEOp struct {
 	derivedReaderID objID
 }
 
-func iteratorPos(i *retryableIter) string {
+func iteratorPos(kf KeyFormat, i *retryableIter) string {
 	var buf bytes.Buffer
-	fmt.Fprintf(&buf, "%q", i.Key())
+	fmt.Fprintf(&buf, "%q", kf.FormatKey(i.Key()))
 	hasPoint, hasRange := i.HasPointAndRange()
 	if hasPoint {
 		fmt.Fprintf(&buf, ",%q", i.Value())
@@ -1457,12 +1467,12 @@ func iteratorPos(i *retryableIter) string {
 	}
 	if hasRange {
 		start, end := i.RangeBounds()
-		fmt.Fprintf(&buf, ",[%q,%q)=>{", start, end)
+		fmt.Fprintf(&buf, ",[%q,%q)=>{", kf.FormatKey(start), kf.FormatKey(end))
 		for i, rk := range i.RangeKeys() {
 			if i > 0 {
 				fmt.Fprint(&buf, ",")
 			}
-			fmt.Fprintf(&buf, "%q=%q", rk.Suffix, rk.Value)
+			fmt.Fprintf(&buf, "%q=%q", kf.FormatKeySuffix(rk.Suffix), rk.Value)
 		}
 		fmt.Fprint(&buf, "}")
 	} else {
@@ -1503,7 +1513,7 @@ func (o *iterSeekGEOp) run(t *Test, h historyRecorder) {
 	}
 	if valid {
 		h.Recordf("%s // [%s,%s] %v", o.formattedString(t.testOpts.KeyFormat),
-			validStr, iteratorPos(i), i.Error())
+			validStr, iteratorPos(t.testOpts.KeyFormat, i), i.Error())
 	} else {
 		h.Recordf("%s // [%s] %v",
 			o.formattedString(t.testOpts.KeyFormat), validStr, i.Error())
@@ -1548,7 +1558,7 @@ func (o *iterSeekPrefixGEOp) run(t *Test, h historyRecorder) {
 	valid := i.SeekPrefixGE(o.key)
 	if valid {
 		h.Recordf("%s // [%t,%s] %v", o.formattedString(t.testOpts.KeyFormat),
-			valid, iteratorPos(i), i.Error())
+			valid, iteratorPos(t.testOpts.KeyFormat, i), i.Error())
 	} else {
 		h.Recordf("%s // [%t] %v", o.formattedString(t.testOpts.KeyFormat), valid, i.Error())
 	}
@@ -1589,7 +1599,7 @@ func (o *iterSeekLTOp) run(t *Test, h historyRecorder) {
 	}
 	if valid {
 		h.Recordf("%s // [%s,%s] %v", o.formattedString(t.testOpts.KeyFormat),
-			validStr, iteratorPos(i), i.Error())
+			validStr, iteratorPos(t.testOpts.KeyFormat, i), i.Error())
 	} else {
 		h.Recordf("%s // [%s] %v", o.formattedString(t.testOpts.KeyFormat),
 			validStr, i.Error())
@@ -1624,7 +1634,7 @@ func (o *iterFirstOp) run(t *Test, h historyRecorder) {
 	valid := i.First()
 	if valid {
 		h.Recordf("%s // [%t,%s] %v", o.formattedString(t.testOpts.KeyFormat),
-			valid, iteratorPos(i), i.Error())
+			valid, iteratorPos(t.testOpts.KeyFormat, i), i.Error())
 	} else {
 		h.Recordf("%s // [%t] %v", o.formattedString(t.testOpts.KeyFormat),
 			valid, i.Error())
@@ -1650,7 +1660,7 @@ func (o *iterLastOp) run(t *Test, h historyRecorder) {
 	valid := i.Last()
 	if valid {
 		h.Recordf("%s // [%t,%s] %v", o.formattedString(t.testOpts.KeyFormat),
-			valid, iteratorPos(i), i.Error())
+			valid, iteratorPos(t.testOpts.KeyFormat, i), i.Error())
 	} else {
 		h.Recordf("%s // [%t] %v", o.formattedString(t.testOpts.KeyFormat),
 			valid, i.Error())
@@ -1684,7 +1694,7 @@ func (o *iterNextOp) run(t *Test, h historyRecorder) {
 	}
 	if valid {
 		h.Recordf("%s // [%s,%s] %v", o.formattedString(t.testOpts.KeyFormat),
-			validStr, iteratorPos(i), i.Error())
+			validStr, iteratorPos(t.testOpts.KeyFormat, i), i.Error())
 	} else {
 		h.Recordf("%s // [%s] %v", o.formattedString(t.testOpts.KeyFormat),
 			validStr, i.Error())
@@ -1713,7 +1723,7 @@ func (o *iterNextPrefixOp) run(t *Test, h historyRecorder) {
 	validStr := validBoolToStr(valid)
 	if valid {
 		h.Recordf("%s // [%s,%s] %v", o.formattedString(t.testOpts.KeyFormat),
-			validStr, iteratorPos(i), i.Error())
+			validStr, iteratorPos(t.testOpts.KeyFormat, i), i.Error())
 	} else {
 		h.Recordf("%s // [%s] %v", o.formattedString(t.testOpts.KeyFormat),
 			validStr, i.Error())
@@ -1784,7 +1794,7 @@ func (o *iterPrevOp) run(t *Test, h historyRecorder) {
 	}
 	if valid {
 		h.Recordf("%s // [%s,%s] %v", o.formattedString(t.testOpts.KeyFormat),
-			validStr, iteratorPos(i), i.Error())
+			validStr, iteratorPos(t.testOpts.KeyFormat, i), i.Error())
 	} else {
 		h.Recordf("%s // [%s] %v", o.formattedString(t.testOpts.KeyFormat),
 			validStr, i.Error())
@@ -2056,12 +2066,17 @@ func (r *replicateOp) runExternalReplicate(
 			if err != nil {
 				panic(err)
 			}
+			t.opts.Comparer.ValidateKey.MustValidate(key.UserKey)
 			return w.Raw().AddWithForceObsolete(base.MakeInternalKey(key.UserKey, 0, key.Kind()), val, false)
 		},
 		func(start, end []byte, seqNum base.SeqNum) error {
+			t.opts.Comparer.ValidateKey.MustValidate(start)
+			t.opts.Comparer.ValidateKey.MustValidate(end)
 			return w.DeleteRange(start, end)
 		},
 		func(start, end []byte, keys []keyspan.Key) error {
+			t.opts.Comparer.ValidateKey.MustValidate(start)
+			t.opts.Comparer.ValidateKey.MustValidate(end)
 			return w.Raw().EncodeSpan(keyspan.Span{
 				Start: start,
 				End:   end,
@@ -2104,6 +2119,9 @@ func (r *replicateOp) runExternalReplicate(
 		return
 	}
 
+	t.opts.Comparer.ValidateKey.MustValidate(r.start)
+	t.opts.Comparer.ValidateKey.MustValidate(r.end)
+
 	_, err = dest.IngestAndExcise(context.Background(), []string{sstPath}, nil, externalSSTs /* external */, pebble.KeyRange{Start: r.start, End: r.end})
 	h.Recordf("%s // %v", r.formattedString(t.testOpts.KeyFormat), err)
 }
@@ -2136,6 +2154,8 @@ func (r *replicateOp) run(t *Test, h historyRecorder) {
 		return
 	}
 
+	t.opts.Comparer.ValidateKey.MustValidate(r.start)
+	t.opts.Comparer.ValidateKey.MustValidate(r.end)
 	// First, do a RangeKeyDelete and DeleteRange on the whole span.
 	if err := dest.RangeKeyDelete(r.start, r.end, t.writeOpts); err != nil {
 		h.Recordf("%s // %v", r.formattedString(t.testOpts.KeyFormat), err)
@@ -2170,6 +2190,8 @@ func (r *replicateOp) run(t *Test, h historyRecorder) {
 			rangeKeys := iter.RangeKeys()
 			rkStart, rkEnd := iter.RangeBounds()
 
+			t.opts.Comparer.ValidateKey.MustValidate(rkStart)
+			t.opts.Comparer.ValidateKey.MustValidate(rkEnd)
 			span := keyspan.Span{Start: rkStart, End: rkEnd, Keys: make([]keyspan.Key, len(rangeKeys))}
 			for i := range rangeKeys {
 				span.Keys[i] = keyspan.Key{
diff --git a/metamorphic/options.go b/metamorphic/options.go
index b10e08a0..3b34aa9e 100644
--- a/metamorphic/options.go
+++ b/metamorphic/options.go
@@ -21,7 +21,6 @@ import (
 	"github.com/cockroachdb/pebble"
 	"github.com/cockroachdb/pebble/bloom"
 	"github.com/cockroachdb/pebble/internal/base"
-	"github.com/cockroachdb/pebble/internal/cache"
 	"github.com/cockroachdb/pebble/objstorage/remote"
 	"github.com/cockroachdb/pebble/sstable"
 	"github.com/cockroachdb/pebble/sstable/block"
@@ -48,7 +47,6 @@ func parseOptions(
 	opts *TestOptions, data string, customOptionParsers map[string]func(string) (CustomOption, bool),
 ) error {
 	hooks := &pebble.ParseHooks{
-		NewCache: pebble.NewCache,
 		NewComparer: func(name string) (*pebble.Comparer, error) {
 			for _, kf := range knownKeyFormats {
 				if kf.Comparer.Name == name {
@@ -671,8 +669,8 @@ func RandomOptions(
 		}
 	}
 
-	opts.BytesPerSync = 1 << uint(rng.IntN(28))     // 1B - 256MB
-	opts.Cache = cache.New(1 << uint(rng.IntN(30))) // 1B - 1GB
+	opts.BytesPerSync = 1 << uint(rng.IntN(28)) // 1B - 256MB
+	opts.CacheSize = 1 << uint(rng.IntN(30))    // 1B - 1GB
 	opts.DisableWAL = rng.IntN(2) == 0
 	opts.FlushDelayDeleteRange = time.Millisecond * time.Duration(5*rng.IntN(245)) // 5-250ms
 	opts.FlushDelayRangeKey = time.Millisecond * time.Duration(5*rng.IntN(245))    // 5-250ms
diff --git a/metamorphic/parser.go b/metamorphic/parser.go
index 7256ba09..1da15596 100644
--- a/metamorphic/parser.go
+++ b/metamorphic/parser.go
@@ -93,9 +93,9 @@ func opArgs(op op) (receiverID *objID, targetID *objID, args []interface{}) {
 	case *newIndexedBatchOp:
 		return &t.dbID, &t.batchID, nil
 	case *newIterOp:
-		return &t.readerID, &t.iterID, []interface{}{&t.lower, &t.upper, &t.keyTypes, &t.filterMin, &t.filterMax, &t.useL6Filters, &t.maskSuffix}
+		return &t.readerID, &t.iterID, []interface{}{&t.lower, &t.upper, &t.keyTypes, &t.filterMax, &t.filterMin, &t.useL6Filters, &t.maskSuffix}
 	case *newIterUsingCloneOp:
-		return &t.existingIterID, &t.iterID, []interface{}{&t.refreshBatch, &t.lower, &t.upper, &t.keyTypes, &t.filterMin, &t.filterMax, &t.useL6Filters, &t.maskSuffix}
+		return &t.existingIterID, &t.iterID, []interface{}{&t.refreshBatch, &t.lower, &t.upper, &t.keyTypes, &t.filterMax, &t.filterMin, &t.useL6Filters, &t.maskSuffix}
 	case *newSnapshotOp:
 		return &t.dbID, &t.snapID, []interface{}{&t.bounds}
 	case *newExternalObjOp:
@@ -119,7 +119,7 @@ func opArgs(op op) (receiverID *objID, targetID *objID, args []interface{}) {
 	case *iterSetBoundsOp:
 		return &t.iterID, nil, []interface{}{&t.lower, &t.upper}
 	case *iterSetOptionsOp:
-		return &t.iterID, nil, []interface{}{&t.lower, &t.upper, &t.keyTypes, &t.filterMin, &t.filterMax, &t.useL6Filters, &t.maskSuffix}
+		return &t.iterID, nil, []interface{}{&t.lower, &t.upper, &t.keyTypes, &t.filterMax, &t.filterMin, &t.useL6Filters, &t.maskSuffix}
 	case *singleDeleteOp:
 		return &t.writerID, nil, []interface{}{&t.key, &t.maybeReplaceDelete}
 	case *rangeKeyDeleteOp:
diff --git a/objstorage/objstorageprovider/objiotracing/obj_io_tracing_off.go b/objstorage/objstorageprovider/objiotracing/obj_io_tracing_off.go
index a4923ab6..3857b418 100644
--- a/objstorage/objstorageprovider/objiotracing/obj_io_tracing_off.go
+++ b/objstorage/objstorageprovider/objiotracing/obj_io_tracing_off.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build !pebble_obj_io_tracing
-// +build !pebble_obj_io_tracing
 
 package objiotracing
 
diff --git a/objstorage/objstorageprovider/objiotracing/obj_io_tracing_on.go b/objstorage/objstorageprovider/objiotracing/obj_io_tracing_on.go
index d3f8b1bb..be4c8e74 100644
--- a/objstorage/objstorageprovider/objiotracing/obj_io_tracing_on.go
+++ b/objstorage/objstorageprovider/objiotracing/obj_io_tracing_on.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build pebble_obj_io_tracing
-// +build pebble_obj_io_tracing
 
 package objiotracing
 
diff --git a/objstorage/objstorageprovider/provider.go b/objstorage/objstorageprovider/provider.go
index 953a9d33..389ae247 100644
--- a/objstorage/objstorageprovider/provider.go
+++ b/objstorage/objstorageprovider/provider.go
@@ -488,13 +488,13 @@ func (p *provider) Lookup(
 	if !ok {
 		return objstorage.ObjectMetadata{}, errors.Wrapf(
 			os.ErrNotExist,
-			"file %s (type %d) unknown to the objstorage provider",
-			fileNum, errors.Safe(fileType),
+			"file %s (type %s) unknown to the objstorage provider",
+			fileNum, fileType,
 		)
 	}
 	if meta.FileType != fileType {
 		return objstorage.ObjectMetadata{}, base.AssertionFailedf(
-			"file %s type mismatch (known type %d, expected type %d)",
+			"file %s type mismatch (known type %s, expected type %s)",
 			fileNum, errors.Safe(meta.FileType), errors.Safe(fileType),
 		)
 	}
@@ -549,8 +549,8 @@ func (p *provider) CheckpointState(
 		if _, ok := p.mu.knownObjects[fileNums[i]]; !ok {
 			return errors.Wrapf(
 				os.ErrNotExist,
-				"file %s (type %d) unknown to the objstorage provider",
-				fileNums[i], errors.Safe(fileType),
+				"file %s (type %s) unknown to the objstorage provider",
+				fileNums[i], fileType,
 			)
 		}
 		// Prevent this object from deletion, at least for the life of this instance.
diff --git a/objstorage/objstorageprovider/vfs.go b/objstorage/objstorageprovider/vfs.go
index 6e842dce..57285315 100644
--- a/objstorage/objstorageprovider/vfs.go
+++ b/objstorage/objstorageprovider/vfs.go
@@ -73,12 +73,15 @@ func (p *provider) vfsInit() error {
 
 	for _, filename := range listing {
 		fileType, fileNum, ok := base.ParseFilename(p.st.FS, filename)
-		if ok && fileType == base.FileTypeTable {
-			o := objstorage.ObjectMetadata{
-				FileType:    fileType,
-				DiskFileNum: fileNum,
+		if ok {
+			switch fileType {
+			case base.FileTypeTable, base.FileTypeBlob:
+				o := objstorage.ObjectMetadata{
+					FileType:    fileType,
+					DiskFileNum: fileNum,
+				}
+				p.mu.knownObjects[o.DiskFileNum] = o
 			}
-			p.mu.knownObjects[o.DiskFileNum] = o
 		}
 	}
 	return nil
diff --git a/obsolete_files.go b/obsolete_files.go
index d1fdabee..e2e76817 100644
--- a/obsolete_files.go
+++ b/obsolete_files.go
@@ -64,7 +64,7 @@ type deletableFile struct {
 
 // obsoleteFile holds information about a file that needs to be deleted soon.
 type obsoleteFile struct {
-	fileType fileType
+	fileType base.FileType
 	// nonLogFile is populated when fileType != fileTypeLog.
 	nonLogFile deletableFile
 	// logFile is populated when fileType == fileTypeLog.
@@ -164,17 +164,20 @@ func (cm *cleanupManager) mainLoop() {
 	for job := range cm.jobsCh {
 		for _, of := range job.obsoleteFiles {
 			switch of.fileType {
-			case fileTypeTable:
+			case base.FileTypeTable:
 				cm.maybePace(&tb, of.fileType, of.nonLogFile.fileNum, of.nonLogFile.fileSize)
 				cm.onTableDeleteFn(of.nonLogFile.fileSize, of.nonLogFile.isLocal)
-				cm.deleteObsoleteObject(fileTypeTable, job.jobID, of.nonLogFile.fileNum)
-			case fileTypeLog:
-				cm.deleteObsoleteFile(of.logFile.FS, fileTypeLog, job.jobID, of.logFile.Path,
-					base.DiskFileNum(of.logFile.NumWAL), of.logFile.ApproxFileSize)
+				cm.deleteObsoleteObject(of.fileType, job.jobID, of.nonLogFile.fileNum)
+			case base.FileTypeBlob:
+				cm.maybePace(&tb, of.fileType, of.nonLogFile.fileNum, of.nonLogFile.fileSize)
+				cm.deleteObsoleteObject(of.fileType, job.jobID, of.nonLogFile.fileNum)
+			case base.FileTypeLog:
+				cm.deleteObsoleteFile(of.logFile.FS, base.FileTypeLog, job.jobID, of.logFile.Path,
+					base.DiskFileNum(of.logFile.NumWAL))
 			default:
 				path := base.MakeFilepath(cm.opts.FS, of.nonLogFile.dir, of.fileType, of.nonLogFile.fileNum)
 				cm.deleteObsoleteFile(
-					cm.opts.FS, of.fileType, job.jobID, path, of.nonLogFile.fileNum, of.nonLogFile.fileSize)
+					cm.opts.FS, of.fileType, job.jobID, path, of.nonLogFile.fileNum)
 			}
 		}
 		cm.mu.Lock()
@@ -187,7 +190,7 @@ func (cm *cleanupManager) mainLoop() {
 
 // fileNumIfSST is read iff fileType is fileTypeTable.
 func (cm *cleanupManager) needsPacing(fileType base.FileType, fileNumIfSST base.DiskFileNum) bool {
-	if fileType != fileTypeTable {
+	if fileType != base.FileTypeTable && fileType != base.FileTypeBlob {
 		return false
 	}
 	meta, err := cm.objProvider.Lookup(fileType, fileNumIfSST)
@@ -229,7 +232,7 @@ func (cm *cleanupManager) maybePace(
 
 // deleteObsoleteFile deletes a (non-object) file that is no longer needed.
 func (cm *cleanupManager) deleteObsoleteFile(
-	fs vfs.FS, fileType fileType, jobID JobID, path string, fileNum base.DiskFileNum, fileSize uint64,
+	fs vfs.FS, fileType base.FileType, jobID JobID, path string, fileNum base.DiskFileNum,
 ) {
 	// TODO(peter): need to handle this error, probably by re-adding the
 	// file that couldn't be deleted to one of the obsolete slices map.
@@ -239,29 +242,29 @@ func (cm *cleanupManager) deleteObsoleteFile(
 	}
 
 	switch fileType {
-	case fileTypeLog:
+	case base.FileTypeLog:
 		cm.opts.EventListener.WALDeleted(WALDeleteInfo{
 			JobID:   int(jobID),
 			Path:    path,
 			FileNum: fileNum,
 			Err:     err,
 		})
-	case fileTypeManifest:
+	case base.FileTypeManifest:
 		cm.opts.EventListener.ManifestDeleted(ManifestDeleteInfo{
 			JobID:   int(jobID),
 			Path:    path,
 			FileNum: fileNum,
 			Err:     err,
 		})
-	case fileTypeTable:
+	case base.FileTypeTable, base.FileTypeBlob:
 		panic("invalid deletion of object file")
 	}
 }
 
 func (cm *cleanupManager) deleteObsoleteObject(
-	fileType fileType, jobID JobID, fileNum base.DiskFileNum,
+	fileType base.FileType, jobID JobID, fileNum base.DiskFileNum,
 ) {
-	if fileType != fileTypeTable {
+	if fileType != base.FileTypeTable && fileType != base.FileTypeBlob {
 		panic("not an object")
 	}
 
@@ -278,13 +281,14 @@ func (cm *cleanupManager) deleteObsoleteObject(
 	}
 
 	switch fileType {
-	case fileTypeTable:
+	case base.FileTypeTable:
 		cm.opts.EventListener.TableDeleted(TableDeleteInfo{
 			JobID:   int(jobID),
 			Path:    path,
 			FileNum: fileNum,
 			Err:     err,
 		})
+		// TODO(jackson): Add BlobFileDeleted event.
 	}
 }
 
@@ -367,7 +371,8 @@ func (d *DB) scanObsoleteFiles(list []string, flushableIngests []*ingestedFlusha
 
 	manifestFileNum := d.mu.versions.manifestFileNum
 
-	var obsoleteTables []tableInfo
+	var obsoleteTables []objectInfo
+	var obsoleteBlobs []objectInfo
 	var obsoleteManifests []fileInfo
 	var obsoleteOptions []fileInfo
 
@@ -377,7 +382,7 @@ func (d *DB) scanObsoleteFiles(list []string, flushableIngests []*ingestedFlusha
 			continue
 		}
 		switch fileType {
-		case fileTypeManifest:
+		case base.FileTypeManifest:
 			if diskFileNum >= manifestFileNum {
 				continue
 			}
@@ -386,7 +391,7 @@ func (d *DB) scanObsoleteFiles(list []string, flushableIngests []*ingestedFlusha
 				fi.FileSize = uint64(stat.Size())
 			}
 			obsoleteManifests = append(obsoleteManifests, fi)
-		case fileTypeOptions:
+		case base.FileTypeOptions:
 			if diskFileNum >= d.optionsFileNum {
 				continue
 			}
@@ -395,7 +400,7 @@ func (d *DB) scanObsoleteFiles(list []string, flushableIngests []*ingestedFlusha
 				fi.FileSize = uint64(stat.Size())
 			}
 			obsoleteOptions = append(obsoleteOptions, fi)
-		case fileTypeTable:
+		case base.FileTypeTable, base.FileTypeBlob:
 			// Objects are handled through the objstorage provider below.
 		default:
 			// Don't delete files we don't know about.
@@ -404,28 +409,32 @@ func (d *DB) scanObsoleteFiles(list []string, flushableIngests []*ingestedFlusha
 
 	objects := d.objProvider.List()
 	for _, obj := range objects {
-		switch obj.FileType {
-		case fileTypeTable:
-			if _, ok := liveFileNums[obj.DiskFileNum]; ok {
-				continue
-			}
-			fileInfo := fileInfo{
-				FileNum: obj.DiskFileNum,
-			}
+		if _, ok := liveFileNums[obj.DiskFileNum]; ok {
+			continue
+		}
+		makeObjectInfo := func() objectInfo {
+			fileInfo := fileInfo{FileNum: obj.DiskFileNum}
 			if size, err := d.objProvider.Size(obj); err == nil {
 				fileInfo.FileSize = uint64(size)
 			}
-			obsoleteTables = append(obsoleteTables, tableInfo{
+			return objectInfo{
 				fileInfo: fileInfo,
 				isLocal:  !obj.IsRemote(),
-			})
+			}
+		}
 
+		switch obj.FileType {
+		case base.FileTypeTable:
+			obsoleteTables = append(obsoleteTables, makeObjectInfo())
+		case base.FileTypeBlob:
+			obsoleteBlobs = append(obsoleteBlobs, makeObjectInfo())
 		default:
 			// Ignore object types we don't know about.
 		}
 	}
 
-	d.mu.versions.obsoleteTables = mergeTableInfos(d.mu.versions.obsoleteTables, obsoleteTables)
+	d.mu.versions.obsoleteTables = mergeObjectInfos(d.mu.versions.obsoleteTables, obsoleteTables)
+	d.mu.versions.obsoleteBlobs = mergeObjectInfos(d.mu.versions.obsoleteBlobs, obsoleteBlobs)
 	d.mu.versions.updateObsoleteTableMetricsLocked()
 	d.mu.versions.obsoleteManifests = merge(d.mu.versions.obsoleteManifests, obsoleteManifests)
 	d.mu.versions.obsoleteOptions = merge(d.mu.versions.obsoleteOptions, obsoleteOptions)
@@ -482,8 +491,10 @@ func (d *DB) deleteObsoleteFiles(jobID JobID) {
 		panic(err)
 	}
 
-	obsoleteTables := append([]tableInfo(nil), d.mu.versions.obsoleteTables...)
-	d.mu.versions.obsoleteTables = nil
+	obsoleteTables := slices.Clone(d.mu.versions.obsoleteTables)
+	d.mu.versions.obsoleteTables = d.mu.versions.obsoleteTables[:0]
+	obsoleteBlobs := slices.Clone(d.mu.versions.obsoleteBlobs)
+	d.mu.versions.obsoleteBlobs = d.mu.versions.obsoleteBlobs[:0]
 
 	for _, tbl := range obsoleteTables {
 		delete(d.mu.versions.zombieTables, tbl.FileNum)
@@ -515,17 +526,32 @@ func (d *DB) deleteObsoleteFiles(jobID JobID) {
 
 	filesToDelete := make([]obsoleteFile, 0, len(obsoleteLogs)+len(obsoleteTables)+len(obsoleteManifests)+len(obsoleteOptions))
 	for _, f := range obsoleteLogs {
-		filesToDelete = append(filesToDelete, obsoleteFile{fileType: fileTypeLog, logFile: f})
+		filesToDelete = append(filesToDelete, obsoleteFile{fileType: base.FileTypeLog, logFile: f})
 	}
 	// We sort to make the order of deletions deterministic, which is nice for
 	// tests.
-	slices.SortFunc(obsoleteTables, func(a, b tableInfo) int {
+	slices.SortFunc(obsoleteTables, func(a, b objectInfo) int {
+		return cmp.Compare(a.FileNum, b.FileNum)
+	})
+	slices.SortFunc(obsoleteBlobs, func(a, b objectInfo) int {
 		return cmp.Compare(a.FileNum, b.FileNum)
 	})
 	for _, f := range obsoleteTables {
 		d.fileCache.evict(f.FileNum)
 		filesToDelete = append(filesToDelete, obsoleteFile{
-			fileType: fileTypeTable,
+			fileType: base.FileTypeTable,
+			nonLogFile: deletableFile{
+				dir:      d.dirname,
+				fileNum:  f.FileNum,
+				fileSize: f.FileSize,
+				isLocal:  f.isLocal,
+			},
+		})
+	}
+	for _, f := range obsoleteBlobs {
+		d.fileCache.evict(f.FileNum)
+		filesToDelete = append(filesToDelete, obsoleteFile{
+			fileType: base.FileTypeBlob,
 			nonLogFile: deletableFile{
 				dir:      d.dirname,
 				fileNum:  f.FileNum,
@@ -534,12 +560,13 @@ func (d *DB) deleteObsoleteFiles(jobID JobID) {
 			},
 		})
 	}
+
 	files := [2]struct {
-		fileType fileType
+		fileType base.FileType
 		obsolete []fileInfo
 	}{
-		{fileTypeManifest, obsoleteManifests},
-		{fileTypeOptions, obsoleteOptions},
+		{base.FileTypeManifest, obsoleteManifests},
+		{base.FileTypeOptions, obsoleteOptions},
 	}
 	for _, f := range files {
 		// We sort to make the order of deletions deterministic, which is nice for
@@ -594,16 +621,16 @@ func merge(a, b []fileInfo) []fileInfo {
 	})
 }
 
-func mergeTableInfos(a, b []tableInfo) []tableInfo {
+func mergeObjectInfos(a, b []objectInfo) []objectInfo {
 	if len(b) == 0 {
 		return a
 	}
 
 	a = append(a, b...)
-	slices.SortFunc(a, func(a, b tableInfo) int {
+	slices.SortFunc(a, func(a, b objectInfo) int {
 		return cmp.Compare(a.FileNum, b.FileNum)
 	})
-	return slices.CompactFunc(a, func(a, b tableInfo) bool {
+	return slices.CompactFunc(a, func(a, b objectInfo) bool {
 		return a.FileNum == b.FileNum
 	})
 }
diff --git a/open.go b/open.go
index 29638c6c..464120d1 100644
--- a/open.go
+++ b/open.go
@@ -32,7 +32,6 @@ import (
 	"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
 	"github.com/cockroachdb/pebble/objstorage/remote"
 	"github.com/cockroachdb/pebble/record"
-	"github.com/cockroachdb/pebble/sstable/block"
 	"github.com/cockroachdb/pebble/vfs"
 	"github.com/cockroachdb/pebble/wal"
 	"github.com/prometheus/client_golang/prometheus"
@@ -78,7 +77,7 @@ func FileCacheSize(maxOpenFiles int) int {
 func Open(dirname string, opts *Options) (db *DB, err error) {
 	// Make a copy of the options so that we don't mutate the passed in options.
 	opts = opts.Clone()
-	opts = opts.EnsureDefaults()
+	opts.EnsureDefaults()
 	if err := opts.Validate(); err != nil {
 		return nil, err
 	}
@@ -206,13 +205,12 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
 	}
 
 	if opts.Cache == nil {
-		opts.Cache = cache.New(cacheDefaultSize)
-	} else {
-		opts.Cache.Ref()
+		opts.Cache = cache.New(opts.CacheSize)
+		defer opts.Cache.Unref()
 	}
 
 	d := &DB{
-		cacheID:             opts.Cache.NewID(),
+		cacheHandle:         opts.Cache.NewHandle(),
 		dirname:             dirname,
 		opts:                opts,
 		cmp:                 opts.Comparer.Compare,
@@ -239,16 +237,10 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
 				d.freeMemTable(obsoleteMemTable)
 			}
 
-			// Release our references to the Cache. Note that both the DB, and
-			// fileCache have a reference. When we release the reference to
-			// the fileCache, and if there are no other references to
-			// the fileCache, then the fileCache will also release its
-			// reference to the cache.
-			opts.Cache.Unref()
-
 			if d.fileCache != nil {
-				_ = d.fileCache.close()
+				_ = d.fileCache.Close()
 			}
+			d.cacheHandle.Close()
 
 			for _, mem := range d.mu.mem.queue {
 				switch t := mem.flushable.(type) {
@@ -403,15 +395,17 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
 
 	if manifestExists && !opts.DisableConsistencyCheck {
 		curVersion := d.mu.versions.currentVersion()
-		if err := checkConsistency(curVersion, dirname, d.objProvider); err != nil {
+		if err := checkConsistency(curVersion, d.objProvider); err != nil {
 			return nil, err
 		}
 	}
 
 	fileCacheSize := FileCacheSize(opts.MaxOpenFiles)
-	d.fileCache = newFileCacheContainer(
-		opts.FileCache, d.cacheID, d.objProvider, d.opts, fileCacheSize,
-		&block.CategoryStatsCollector{})
+	if opts.FileCache == nil {
+		opts.FileCache = NewFileCache(opts.Experimental.FileCacheShards, fileCacheSize)
+		defer opts.FileCache.Unref()
+	}
+	d.fileCache = opts.FileCache.newHandle(d.cacheHandle, d.objProvider, d.opts)
 	d.newIters = d.fileCache.newIters
 	d.tableNewRangeKeyIter = tableNewRangeKeyIter(d.newIters)
 
@@ -419,14 +413,14 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
 		return true
 	})
 	d.mu.annotators.remoteSize = d.makeFileSizeAnnotator(func(f *manifest.FileMetadata) bool {
-		meta, err := d.objProvider.Lookup(fileTypeTable, f.FileBacking.DiskFileNum)
+		meta, err := d.objProvider.Lookup(base.FileTypeTable, f.FileBacking.DiskFileNum)
 		if err != nil {
 			return false
 		}
 		return meta.IsRemote()
 	})
 	d.mu.annotators.externalSize = d.makeFileSizeAnnotator(func(f *manifest.FileMetadata) bool {
-		meta, err := d.objProvider.Lookup(fileTypeTable, f.FileBacking.DiskFileNum)
+		meta, err := d.objProvider.Lookup(base.FileTypeTable, f.FileBacking.DiskFileNum)
 		if err != nil {
 			return false
 		}
@@ -446,14 +440,14 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
 		d.mu.versions.markFileNumUsed(fn)
 
 		switch ft {
-		case fileTypeLog:
+		case base.FileTypeLog:
 			// Ignore.
-		case fileTypeOptions:
+		case base.FileTypeOptions:
 			if previousOptionsFileNum < fn {
 				previousOptionsFileNum = fn
 				previousOptionsFilename = filename
 			}
-		case fileTypeTemp, fileTypeOldTemp:
+		case base.FileTypeTemp, base.FileTypeOldTemp:
 			if !d.opts.ReadOnly {
 				// Some codepaths write to a temporary file and then
 				// rename it to its final location when complete.  A
@@ -568,8 +562,8 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
 
 		// Write the current options to disk.
 		d.optionsFileNum = d.mu.versions.getNextDiskFileNum()
-		tmpPath := base.MakeFilepath(opts.FS, dirname, fileTypeTemp, d.optionsFileNum)
-		optionsPath := base.MakeFilepath(opts.FS, dirname, fileTypeOptions, d.optionsFileNum)
+		tmpPath := base.MakeFilepath(opts.FS, dirname, base.FileTypeTemp, d.optionsFileNum)
+		optionsPath := base.MakeFilepath(opts.FS, dirname, base.FileTypeOptions, d.optionsFileNum)
 
 		// Write them to a temporary file first, in case we crash before
 		// we're done. A corrupt options file prevents opening the
@@ -723,7 +717,7 @@ func GetVersion(dir string, fs vfs.FS) (string, error) {
 			continue
 		}
 		switch ft {
-		case fileTypeOptions:
+		case base.FileTypeOptions:
 			// If this file has a higher number than the last options file
 			// processed, reset version. This is because rocksdb often
 			// writes multiple options files without deleting previous ones.
@@ -813,13 +807,14 @@ func (d *DB) replayIngestedFlushable(
 	meta := make([]*fileMetadata, len(fileNums))
 	var lastRangeKey keyspan.Span
 	for i, n := range fileNums {
-		readable, err := d.objProvider.OpenForReading(context.TODO(), fileTypeTable, n, objstorage.OpenOptions{MustExist: true})
+		readable, err := d.objProvider.OpenForReading(context.TODO(), base.FileTypeTable, n,
+			objstorage.OpenOptions{MustExist: true})
 		if err != nil {
 			return nil, errors.Wrap(err, "pebble: error when opening flushable ingest files")
 		}
 		// NB: ingestLoad1 will close readable.
 		meta[i], lastRangeKey, err = ingestLoad1(context.TODO(), d.opts, d.FormatMajorVersion(),
-			readable, d.cacheID, base.PhysicalTableFileNum(n), disableRangeKeyChecks())
+			readable, d.cacheHandle, base.PhysicalTableFileNum(n), disableRangeKeyChecks())
 		if err != nil {
 			return nil, errors.Wrap(err, "pebble: error when loading flushable ingest files")
 		}
@@ -941,8 +936,11 @@ func (d *DB) replayWAL(
 			// to otherwise treat them like EOF.
 			if err == io.EOF {
 				break
-			} else if record.IsInvalidRecord(err) && !strictWALTail {
-				break
+			} else if record.IsInvalidRecord(err) {
+				if !strictWALTail {
+					break
+				}
+				err = errors.Mark(err, ErrCorruption)
 			}
 			return nil, 0, errors.Wrap(err, "pebble: error when replaying WAL")
 		}
@@ -1119,7 +1117,7 @@ func Peek(dirname string, fs vfs.FS) (*DBDesc, error) {
 	var previousOptionsFileNum base.DiskFileNum
 	for _, filename := range ls {
 		ft, fn, ok := base.ParseFilename(fs, filename)
-		if !ok || ft != fileTypeOptions || fn < previousOptionsFileNum {
+		if !ok || ft != base.FileTypeOptions || fn < previousOptionsFileNum {
 			continue
 		}
 		previousOptionsFileNum = fn
@@ -1127,7 +1125,7 @@ func Peek(dirname string, fs vfs.FS) (*DBDesc, error) {
 	}
 
 	if exists {
-		desc.ManifestFilename = base.MakeFilepath(fs, dirname, fileTypeManifest, manifestFileNum)
+		desc.ManifestFilename = base.MakeFilepath(fs, dirname, base.FileTypeManifest, manifestFileNum)
 	}
 	return desc, nil
 }
@@ -1140,7 +1138,7 @@ func Peek(dirname string, fs vfs.FS) (*DBDesc, error) {
 // LockDirectory may be used to expand the critical section protected by the
 // database lock to include setup before the call to Open.
 func LockDirectory(dirname string, fs vfs.FS) (*Lock, error) {
-	fileLock, err := fs.Lock(base.MakeFilepath(fs, dirname, fileTypeLock, base.DiskFileNum(0)))
+	fileLock, err := fs.Lock(base.MakeFilepath(fs, dirname, base.FileTypeLock, base.DiskFileNum(0)))
 	if err != nil {
 		return nil, err
 	}
@@ -1238,7 +1236,7 @@ func IsCorruptionError(err error) bool {
 	return errors.Is(err, base.ErrCorruption)
 }
 
-func checkConsistency(v *manifest.Version, dirname string, objProvider objstorage.Provider) error {
+func checkConsistency(v *manifest.Version, objProvider objstorage.Provider) error {
 	var errs []error
 	dedup := make(map[base.DiskFileNum]struct{})
 	for level, files := range v.Levels {
diff --git a/options.go b/options.go
index c991648f..e0568ffd 100644
--- a/options.go
+++ b/options.go
@@ -449,10 +449,7 @@ type LevelOptions struct {
 // EnsureDefaults ensures that the default values for all of the options have
 // been initialized. It is valid to call EnsureDefaults on a nil receiver. A
 // non-nil result will always be returned.
-func (o *LevelOptions) EnsureDefaults() *LevelOptions {
-	if o == nil {
-		o = &LevelOptions{}
-	}
+func (o *LevelOptions) EnsureDefaults() {
 	if o.BlockRestartInterval <= 0 {
 		o.BlockRestartInterval = base.DefaultBlockRestartInterval
 	}
@@ -473,7 +470,6 @@ func (o *LevelOptions) EnsureDefaults() *LevelOptions {
 	if o.TargetFileSize <= 0 {
 		o.TargetFileSize = 2 << 20 // 2 MB
 	}
-	return o
 }
 
 // Options holds the optional parameters for configuring pebble. These options
@@ -489,10 +485,11 @@ type Options struct {
 	// The default value is 512KB.
 	BytesPerSync int
 
-	// Cache is used to cache uncompressed blocks from sstables.
-	//
-	// The default cache size is 8 MB.
+	// Cache is used to cache uncompressed blocks from sstables. If it is nil,
+	// a block cache of CacheSize will be created for each DB.
 	Cache *cache.Cache
+	// CacheSize is used when Cache is not set. The default value is 8 MB.
+	CacheSize int64
 
 	// LoadBlockSema, if set, is used to limit the number of blocks that can be
 	// loaded (i.e. read from the filesystem) in parallel. Each load acquires one
@@ -654,19 +651,6 @@ type Options struct {
 		// limited by runtime.GOMAXPROCS.
 		FileCacheShards int
 
-		// KeyValidationFunc is a function to validate a user key in an SSTable.
-		//
-		// Currently, this function is used to validate the smallest and largest
-		// keys in an SSTable undergoing compaction. In this case, returning an
-		// error from the validation function will result in a panic at runtime,
-		// given that there is rarely any way of recovering from malformed keys
-		// present in compacted files. By default, validation is not performed.
-		//
-		// Additional use-cases may be added in the future.
-		//
-		// NOTE: callers should take care to not mutate the key being validated.
-		KeyValidationFunc func(userKey []byte) error
-
 		// ValidateOnIngest schedules validation of sstables after they have
 		// been ingested.
 		//
@@ -1139,10 +1123,10 @@ func DebugCheckLevels(db *DB) error {
 }
 
 // EnsureDefaults ensures that the default values for all options are set if a
-// valid value was not already specified. Returns the new options.
-func (o *Options) EnsureDefaults() *Options {
-	if o == nil {
-		o = &Options{}
+// valid value was not already specified.
+func (o *Options) EnsureDefaults() {
+	if o.Cache == nil && o.CacheSize == 0 {
+		o.CacheSize = cacheDefaultSize
 	}
 	o.Comparer = o.Comparer.EnsureDefaults()
 
@@ -1165,9 +1149,6 @@ func (o *Options) EnsureDefaults() *Options {
 	if o.Experimental.CompactionLimiter == nil {
 		o.Experimental.CompactionLimiter = &base.DefaultCompactionLimiter{}
 	}
-	if o.Experimental.KeyValidationFunc == nil {
-		o.Experimental.KeyValidationFunc = func([]byte) error { return nil }
-	}
 	if o.KeySchema == "" && len(o.KeySchemas) == 0 {
 		ks := colblk.DefaultKeySchema(o.Comparer, 16 /* bundleSize */)
 		o.KeySchema = ks.Name
@@ -1301,12 +1282,18 @@ func (o *Options) EnsureDefaults() *Options {
 	}
 
 	o.initMaps()
+}
+
+// DefaultOptions returns a new Options object with the default values set.
+func DefaultOptions() *Options {
+	o := &Options{}
+	o.EnsureDefaults()
 	return o
 }
 
 // WithFSDefaults configures the Options to wrap the configured filesystem with
 // the default virtual file system middleware, like disk-health checking.
-func (o *Options) WithFSDefaults() *Options {
+func (o *Options) WithFSDefaults() {
 	if o.FS == nil {
 		o.FS = vfs.Default
 	}
@@ -1314,7 +1301,6 @@ func (o *Options) WithFSDefaults() *Options {
 		func(info vfs.DiskSlowInfo) {
 			o.EventListener.DiskSlow(info)
 		})
-	return o
 }
 
 // AddEventListener adds the provided event listener to the Options, in addition
@@ -1374,7 +1360,7 @@ func filterPolicyName(p FilterPolicy) string {
 func (o *Options) String() string {
 	var buf bytes.Buffer
 
-	cacheSize := int64(cacheDefaultSize)
+	cacheSize := o.CacheSize
 	if o.Cache != nil {
 		cacheSize = o.Cache.MaxSize()
 	}
@@ -1567,7 +1553,6 @@ func parseOptions(s string, fns parseOptionsFuncs) error {
 // ParseHooks contains callbacks to create options fields which can have
 // user-defined implementations.
 type ParseHooks struct {
-	NewCache        func(size int64) *Cache
 	NewCleaner      func(name string) (Cleaner, error)
 	NewComparer     func(name string) (*Comparer, error)
 	NewFilterPolicy func(name string) (FilterPolicy, error)
@@ -1619,16 +1604,7 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
 			case "bytes_per_sync":
 				o.BytesPerSync, err = strconv.Atoi(value)
 			case "cache_size":
-				var n int64
-				n, err = strconv.ParseInt(value, 10, 64)
-				if err == nil && hooks != nil && hooks.NewCache != nil {
-					if o.Cache != nil {
-						o.Cache.Unref()
-					}
-					o.Cache = hooks.NewCache(n)
-				}
-				// We avoid calling cache.New in parsing because it makes it
-				// too easy to leak a cache.
+				o.CacheSize, err = strconv.ParseInt(value, 10, 64)
 			case "cleaner":
 				switch value {
 				case "archive":
@@ -2049,9 +2025,6 @@ func (o *Options) Validate() error {
 		fmt.Fprintf(&buf, "FormatMajorVersion (%d) when CreateOnShared is set must be at least %d\n",
 			o.FormatMajorVersion, FormatMinForSharedObjects)
 	}
-	if o.FileCache != nil && o.Cache != o.FileCache.cache {
-		fmt.Fprintf(&buf, "underlying cache in the FileCache and the Cache dont match\n")
-	}
 	if len(o.KeySchemas) > 0 {
 		if o.KeySchema == "" {
 			fmt.Fprintf(&buf, "KeySchemas is set but KeySchema is not\n")
diff --git a/record/log_writer.go b/record/log_writer.go
index efbf4609..8ea1e17a 100644
--- a/record/log_writer.go
+++ b/record/log_writer.go
@@ -961,7 +961,7 @@ func (w *LogWriter) emitEOFTrailer() {
 	i := b.written.Load()
 	binary.LittleEndian.PutUint32(b.buf[i+0:i+4], 0) // CRC
 	binary.LittleEndian.PutUint16(b.buf[i+4:i+6], 0) // Size
-	b.buf[i+6] = recyclableFullChunkType
+	b.buf[i+6] = recyclableFullChunkEncoding
 	binary.LittleEndian.PutUint32(b.buf[i+7:i+11], w.logNum+1) // Log number
 	b.written.Store(i + int32(recyclableHeaderSize))
 }
@@ -974,15 +974,15 @@ func (w *LogWriter) emitFragment(n int, p []byte) (remainingP []byte) {
 
 	if last {
 		if first {
-			b.buf[i+6] = recyclableFullChunkType
+			b.buf[i+6] = recyclableFullChunkEncoding
 		} else {
-			b.buf[i+6] = recyclableLastChunkType
+			b.buf[i+6] = recyclableLastChunkEncoding
 		}
 	} else {
 		if first {
-			b.buf[i+6] = recyclableFirstChunkType
+			b.buf[i+6] = recyclableFirstChunkEncoding
 		} else {
-			b.buf[i+6] = recyclableMiddleChunkType
+			b.buf[i+6] = recyclableMiddleChunkEncoding
 		}
 	}
 
diff --git a/record/record.go b/record/record.go
index 7b6d5258..cfdfeb81 100644
--- a/record/record.go
+++ b/record/record.go
@@ -115,15 +115,17 @@ import (
 
 // These constants are part of the wire format and should not be changed.
 const (
-	fullChunkType   = 1
-	firstChunkType  = 2
-	middleChunkType = 3
-	lastChunkType   = 4
-
-	recyclableFullChunkType   = 5
-	recyclableFirstChunkType  = 6
-	recyclableMiddleChunkType = 7
-	recyclableLastChunkType   = 8
+	invalidChunkEncoding = 0
+
+	fullChunkEncoding   = 1
+	firstChunkEncoding  = 2
+	middleChunkEncoding = 3
+	lastChunkEncoding   = 4
+
+	recyclableFullChunkEncoding   = 5
+	recyclableFirstChunkEncoding  = 6
+	recyclableMiddleChunkEncoding = 7
+	recyclableLastChunkEncoding   = 8
 )
 
 const (
@@ -133,6 +135,57 @@ const (
 	recyclableHeaderSize = legacyHeaderSize + 4
 )
 
+// chunkPosition represents the type of a chunk in the WAL.
+// Records can be split into multiple chunks and marked
+// by the following position types:
+// - invalidChunkPosition: an invalid chunk
+// - fullChunkPosition: a complete record stored in a single chunk
+// - firstChunkPosition: first chunk of a multi-chunk record
+// - middleChunkPosition: intermediate chunk in a multi-chunk record
+// - lastChunkPosition: final chunk of a multi-chunk record
+type chunkPosition int
+
+const (
+	invalidChunkPosition chunkPosition = iota
+	fullChunkPosition
+	firstChunkPosition
+	middleChunkPosition
+	lastChunkPosition
+)
+
+// wireFormat specifies the encoding format used for chunks.
+// wireFormat is used for backwards compatibility and new
+// wire formats may be introduced to support additional
+// WAL chunks.
+type wireFormat int
+
+const (
+	invalidWireFormat wireFormat = iota
+	legacyWireFormat
+	recyclableWireFormat
+)
+
+// headerFormat represents the format of a chunk which has
+// a chunkPosition, wireFormat, and a headerSize.
+type headerFormat struct {
+	chunkPosition
+	wireFormat
+	headerSize int
+}
+
+// headerFormatMappings translates encodings to headerFormats
+var headerFormatMappings = [...]headerFormat{
+	invalidChunkEncoding:          {chunkPosition: invalidChunkPosition, wireFormat: invalidWireFormat, headerSize: 0},
+	fullChunkEncoding:             {chunkPosition: fullChunkPosition, wireFormat: legacyWireFormat, headerSize: legacyHeaderSize},
+	firstChunkEncoding:            {chunkPosition: firstChunkPosition, wireFormat: legacyWireFormat, headerSize: legacyHeaderSize},
+	middleChunkEncoding:           {chunkPosition: middleChunkPosition, wireFormat: legacyWireFormat, headerSize: legacyHeaderSize},
+	lastChunkEncoding:             {chunkPosition: lastChunkPosition, wireFormat: legacyWireFormat, headerSize: legacyHeaderSize},
+	recyclableFullChunkEncoding:   {chunkPosition: fullChunkPosition, wireFormat: recyclableWireFormat, headerSize: recyclableHeaderSize},
+	recyclableFirstChunkEncoding:  {chunkPosition: firstChunkPosition, wireFormat: recyclableWireFormat, headerSize: recyclableHeaderSize},
+	recyclableMiddleChunkEncoding: {chunkPosition: middleChunkPosition, wireFormat: recyclableWireFormat, headerSize: recyclableHeaderSize},
+	recyclableLastChunkEncoding:   {chunkPosition: lastChunkPosition, wireFormat: recyclableWireFormat, headerSize: recyclableHeaderSize},
+}
+
 var (
 	// ErrNotAnIOSeeker is returned if the io.Reader underlying a Reader does not implement io.Seeker.
 	ErrNotAnIOSeeker = errors.New("pebble/record: reader does not implement io.Seeker")
@@ -142,12 +195,12 @@ var (
 
 	// ErrZeroedChunk is returned if a chunk is encountered that is zeroed. This
 	// usually occurs due to log file preallocation.
-	ErrZeroedChunk = base.CorruptionErrorf("pebble/record: zeroed chunk")
+	ErrZeroedChunk = errors.New("pebble/record: zeroed chunk")
 
 	// ErrInvalidChunk is returned if a chunk is encountered with an invalid
 	// header, length, or checksum. This usually occurs when a log is recycled,
 	// but can also occur due to corruption.
-	ErrInvalidChunk = base.CorruptionErrorf("pebble/record: invalid chunk")
+	ErrInvalidChunk = errors.New("pebble/record: invalid chunk")
 )
 
 // IsInvalidRecord returns true if the error matches one of the error types
@@ -202,9 +255,15 @@ func (r *Reader) nextChunk(wantFirst bool) error {
 		if r.end+legacyHeaderSize <= r.n {
 			checksum := binary.LittleEndian.Uint32(r.buf[r.end+0 : r.end+4])
 			length := binary.LittleEndian.Uint16(r.buf[r.end+4 : r.end+6])
-			chunkType := r.buf[r.end+6]
+			chunkEncoding := r.buf[r.end+6]
+
+			if int(chunkEncoding) >= len(headerFormatMappings) {
+				return ErrInvalidChunk
+			}
+			headerFormat := headerFormatMappings[chunkEncoding]
+			chunkPosition, wireFormat, headerSize := headerFormat.chunkPosition, headerFormat.wireFormat, headerFormat.headerSize
 
-			if checksum == 0 && length == 0 && chunkType == 0 {
+			if checksum == 0 && length == 0 && chunkPosition == invalidChunkPosition {
 				if r.end+recyclableHeaderSize > r.n {
 					// Skip the rest of the block if the recyclable header size does not
 					// fit within it.
@@ -223,26 +282,25 @@ func (r *Reader) nextChunk(wantFirst bool) error {
 				return ErrZeroedChunk
 			}
 
-			headerSize := legacyHeaderSize
-			if chunkType >= recyclableFullChunkType && chunkType <= recyclableLastChunkType {
-				headerSize = recyclableHeaderSize
+			if wireFormat == invalidWireFormat {
+				return ErrInvalidChunk
+			}
+			if wireFormat == recyclableWireFormat {
 				if r.end+headerSize > r.n {
 					return ErrInvalidChunk
 				}
 
 				logNum := binary.LittleEndian.Uint32(r.buf[r.end+7 : r.end+11])
 				if logNum != r.logNum {
-					if wantFirst {
-						// If we're looking for the first chunk of a record, we can treat a
-						// previous instance of the log as EOF.
+					// An EOF trailer encodes a log number that is 1 more than the
+					// current log number.
+					if logNum == 1+r.logNum && wantFirst {
 						return io.EOF
 					}
 					// Otherwise, treat this chunk as invalid in order to prevent reading
 					// of a partial record.
 					return ErrInvalidChunk
 				}
-
-				chunkType -= (recyclableFullChunkType - 1)
 			}
 
 			r.begin = r.end + headerSize
@@ -263,11 +321,11 @@ func (r *Reader) nextChunk(wantFirst bool) error {
 				return ErrInvalidChunk
 			}
 			if wantFirst {
-				if chunkType != fullChunkType && chunkType != firstChunkType {
+				if chunkPosition != fullChunkPosition && chunkPosition != firstChunkPosition {
 					continue
 				}
 			}
-			r.last = chunkType == fullChunkType || chunkType == lastChunkType
+			r.last = chunkPosition == fullChunkPosition || chunkPosition == lastChunkPosition
 			r.recovering = false
 			return nil
 		}
@@ -469,15 +527,15 @@ func (w *Writer) fillHeader(last bool) {
 	}
 	if last {
 		if w.first {
-			w.buf[w.i+6] = fullChunkType
+			w.buf[w.i+6] = fullChunkEncoding
 		} else {
-			w.buf[w.i+6] = lastChunkType
+			w.buf[w.i+6] = lastChunkEncoding
 		}
 	} else {
 		if w.first {
-			w.buf[w.i+6] = firstChunkType
+			w.buf[w.i+6] = firstChunkEncoding
 		} else {
-			w.buf[w.i+6] = middleChunkType
+			w.buf[w.i+6] = middleChunkEncoding
 		}
 	}
 	binary.LittleEndian.PutUint32(w.buf[w.i+0:w.i+4], crc.New(w.buf[w.i+6:w.j]).Value())
diff --git a/scan_internal.go b/scan_internal.go
index fd43511d..3e674ac8 100644
--- a/scan_internal.go
+++ b/scan_internal.go
@@ -709,7 +709,7 @@ func scanInternalImpl(
 
 				var objMeta objstorage.ObjectMetadata
 				var err error
-				objMeta, err = provider.Lookup(fileTypeTable, f.FileBacking.DiskFileNum)
+				objMeta, err = provider.Lookup(base.FileTypeTable, f.FileBacking.DiskFileNum)
 				if err != nil {
 					return err
 				}
@@ -937,7 +937,7 @@ func (i *scanInternalIterator) constructPointIter(
 		if level == skipStart {
 			nonRemoteFiles := make([]*manifest.FileMetadata, 0)
 			for f := levIter.First(); f != nil; f = levIter.Next() {
-				meta, err := i.db.objProvider.Lookup(fileTypeTable, f.FileBacking.DiskFileNum)
+				meta, err := i.db.objProvider.Lookup(base.FileTypeTable, f.FileBacking.DiskFileNum)
 				if err != nil {
 					return err
 				}
@@ -1049,7 +1049,7 @@ func (i *scanInternalIterator) constructRangeKeyIter() error {
 		if level == skipStart {
 			nonRemoteFiles := make([]*manifest.FileMetadata, 0)
 			for f := levIter.First(); f != nil; f = levIter.Next() {
-				meta, err := i.db.objProvider.Lookup(fileTypeTable, f.FileBacking.DiskFileNum)
+				meta, err := i.db.objProvider.Lookup(base.FileTypeTable, f.FileBacking.DiskFileNum)
 				if err != nil {
 					return err
 				}
diff --git a/sstable/block/block.go b/sstable/block/block.go
index 06993e45..239ac05e 100644
--- a/sstable/block/block.go
+++ b/sstable/block/block.go
@@ -323,7 +323,7 @@ type ReadEnv struct {
 	// iterator is closed. In the important code paths, the CategoryStatsCollector
 	// is managed by the fileCacheContainer.
 	Stats     *base.InternalIteratorStats
-	IterStats IterStatsAccumulator
+	IterStats *CategoryStatsShard
 
 	// BufferPool is not-nil if we read blocks into a buffer pool and not into the
 	// cache. This is used during compactions.
@@ -378,14 +378,6 @@ func (r *Reader) Init(readable objstorage.Readable, ro ReaderOptions, checksumTy
 	r.readable = readable
 	r.opts = ro
 	r.checksumType = checksumType
-	if r.opts.CacheOpts.Cache == nil {
-		r.opts.CacheOpts.Cache = cache.New(0)
-	} else {
-		r.opts.CacheOpts.Cache.Ref()
-	}
-	if r.opts.CacheOpts.CacheID == 0 {
-		r.opts.CacheOpts.CacheID = r.opts.CacheOpts.Cache.NewID()
-	}
 }
 
 // FileNum returns the file number of the file being read.
@@ -407,64 +399,60 @@ func (r *Reader) Read(
 	bh Handle,
 	initBlockMetadataFn func(*Metadata, []byte) error,
 ) (handle BufferHandle, _ error) {
-	var cv *cache.Value
-	var crh cache.ReadHandle
-	hit := true
-	if env.BufferPool == nil {
-		var errorDuration time.Duration
-		var err error
-		cv, crh, errorDuration, hit, err = r.opts.CacheOpts.Cache.GetWithReadHandle(
-			ctx, r.opts.CacheOpts.CacheID, r.opts.CacheOpts.FileNum, bh.Offset)
-		if errorDuration > 5*time.Millisecond && r.opts.LoggerAndTracer.IsTracingEnabled(ctx) {
-			r.opts.LoggerAndTracer.Eventf(
-				ctx, "waited for turn when %s time wasted by failed reads", errorDuration.String())
+	// The compaction path uses env.BufferPool, and does not coordinate read
+	// using a cache.ReadHandle. This is ok since only a single compaction is
+	// reading a block.
+	if r.opts.CacheOpts.CacheHandle == nil || env.BufferPool != nil {
+		if r.opts.CacheOpts.CacheHandle != nil {
+			if cv := r.opts.CacheOpts.CacheHandle.Get(r.opts.CacheOpts.FileNum, bh.Offset); cv != nil {
+				recordCacheHit(ctx, env, readHandle, bh)
+				return CacheBufferHandle(cv), nil
+			}
 		}
-		// TODO(sumeer): consider tracing when waited longer than some duration
-		// for turn to do the read.
+		value, err := r.doRead(ctx, env, readHandle, bh, initBlockMetadataFn)
 		if err != nil {
 			return BufferHandle{}, err
 		}
-	} else {
-		// The compaction path uses env.BufferPool, and does not coordinate read
-		// using a cache.ReadHandle. This is ok since only a single compaction is
-		// reading a block.
-		cv = r.opts.CacheOpts.Cache.Get(r.opts.CacheOpts.CacheID, r.opts.CacheOpts.FileNum, bh.Offset)
-		if cv != nil {
-			hit = true
-		}
+		return value.MakeHandle(), err
+	}
+
+	cv, crh, errorDuration, hit, err := r.opts.CacheOpts.CacheHandle.GetWithReadHandle(
+		ctx, r.opts.CacheOpts.FileNum, bh.Offset)
+	if errorDuration > 5*time.Millisecond && r.opts.LoggerAndTracer.IsTracingEnabled(ctx) {
+		r.opts.LoggerAndTracer.Eventf(
+			ctx, "waited for turn when %s time wasted by failed reads", errorDuration.String())
 	}
-	// INVARIANT: hit => cv != nil
+	// TODO(sumeer): consider tracing when waited longer than some duration
+	// for turn to do the read.
+	if err != nil {
+		return BufferHandle{}, err
+	}
+
 	if cv != nil {
-		if hit {
-			// Cache hit.
-			if readHandle != nil {
-				readHandle.RecordCacheHit(ctx, int64(bh.Offset), int64(bh.Length+TrailerLen))
-			}
-			env.BlockServedFromCache(bh.Length)
-		}
 		if invariants.Enabled && crh.Valid() {
 			panic("cache.ReadHandle must not be valid")
 		}
+		if hit {
+			recordCacheHit(ctx, env, readHandle, bh)
+		}
 		return CacheBufferHandle(cv), nil
 	}
 
-	// Need to read. First acquire loadBlockSema, if needed.
-	if sema := r.opts.LoadBlockSema; sema != nil {
-		if err := sema.Acquire(ctx, 1); err != nil {
-			// An error here can only come from the context.
-			return BufferHandle{}, err
-		}
-		defer sema.Release(1)
-	}
 	value, err := r.doRead(ctx, env, readHandle, bh, initBlockMetadataFn)
 	if err != nil {
-		if crh.Valid() {
-			crh.SetReadError(err)
-		}
+		crh.SetReadError(err)
 		return BufferHandle{}, err
 	}
-	h := value.MakeHandle(crh, r.opts.CacheOpts.CacheID, r.opts.CacheOpts.FileNum, bh.Offset)
-	return h, nil
+	crh.SetReadValue(value.v)
+	return value.MakeHandle(), nil
+}
+
+func recordCacheHit(ctx context.Context, env ReadEnv, readHandle objstorage.ReadHandle, bh Handle) {
+	// Cache hit.
+	if readHandle != nil {
+		readHandle.RecordCacheHit(ctx, int64(bh.Offset), int64(bh.Length+TrailerLen))
+	}
+	env.BlockServedFromCache(bh.Length)
 }
 
 // TODO(sumeer): should the threshold be configurable.
@@ -479,6 +467,15 @@ func (r *Reader) doRead(
 	bh Handle,
 	initBlockMetadataFn func(*Metadata, []byte) error,
 ) (Value, error) {
+	// First acquire loadBlockSema, if needed.
+	if sema := r.opts.LoadBlockSema; sema != nil {
+		if err := sema.Acquire(ctx, 1); err != nil {
+			// An error here can only come from the context.
+			return Value{}, err
+		}
+		defer sema.Release(1)
+	}
+
 	compressed := Alloc(int(bh.Length+TrailerLen), env.BufferPool)
 	readStopwatch := makeStopwatch()
 	var err error
@@ -548,7 +545,7 @@ func (r *Reader) Readable() objstorage.Readable {
 // Users should prefer using Read, which handles reading from object storage on
 // a cache miss.
 func (r *Reader) GetFromCache(bh Handle) *cache.Value {
-	return r.opts.CacheOpts.Cache.Get(r.opts.CacheOpts.CacheID, r.opts.CacheOpts.FileNum, bh.Offset)
+	return r.opts.CacheOpts.CacheHandle.Get(r.opts.CacheOpts.FileNum, bh.Offset)
 }
 
 // UsePreallocatedReadHandle returns a ReadHandle that reads from the reader and
@@ -562,7 +559,6 @@ func (r *Reader) UsePreallocatedReadHandle(
 
 // Close releases resources associated with the Reader.
 func (r *Reader) Close() error {
-	r.opts.CacheOpts.Cache.Unref()
 	var err error
 	if r.readable != nil {
 		err = r.readable.Close()
diff --git a/sstable/block/buffer_pool.go b/sstable/block/buffer_pool.go
index 0da0d8a6..b4b5dabe 100644
--- a/sstable/block/buffer_pool.go
+++ b/sstable/block/buffer_pool.go
@@ -8,7 +8,6 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/cache"
-	"github.com/cockroachdb/pebble/internal/invariants"
 )
 
 // Alloc allocates a new Value for a block of length n (excluding the block
@@ -44,34 +43,24 @@ func (b Value) BlockData() []byte {
 	return b.getInternalBuf()[MetadataSize:]
 }
 
-// GetMetadata returns the block metadata.
+// BlockMetadata returns the block metadata.
 func (b Value) BlockMetadata() *Metadata {
 	return (*Metadata)(b.getInternalBuf())
 }
 
-// MakeHandle constructs a BufferHandle from the Value. If the Value is not
-// backed by a buffer pool, MakeHandle inserts the value into the block cache,
-// returning a handle to the now resident value.
-func (b Value) MakeHandle(
-	crh cache.ReadHandle, cacheID cache.ID, fileNum base.DiskFileNum, offset uint64,
-) BufferHandle {
+// MakeHandle constructs a BufferHandle from the Value.
+func (b Value) MakeHandle() BufferHandle {
 	if b.buf.Valid() {
-		if invariants.Enabled && crh.Valid() {
-			panic("cache.ReadHandle was valid")
-		}
 		return BufferHandle{b: b.buf}
 	}
-	crh.SetReadValue(b.v)
 	return BufferHandle{cv: b.v}
 }
 
-func (b *Value) SetInCacheForTesting(
-	c *cache.Cache, cacheID cache.ID, fileNum base.DiskFileNum, offset uint64,
-) {
+func (b *Value) SetInCacheForTesting(h *cache.Handle, fileNum base.DiskFileNum, offset uint64) {
 	if b.buf.Valid() {
 		panic("block value must be backed by a cache.Value")
 	}
-	c.Set(cacheID, fileNum, offset, b.v)
+	h.Set(fileNum, offset, b.v)
 	b.v.Release()
 	b.v = nil
 }
diff --git a/sstable/block/category_stats.go b/sstable/block/category_stats.go
index a516371f..abe7cb9b 100644
--- a/sstable/block/category_stats.go
+++ b/sstable/block/category_stats.go
@@ -6,6 +6,7 @@ package block
 
 import (
 	"cmp"
+	"runtime"
 	"slices"
 	"sync"
 	"sync/atomic"
@@ -32,6 +33,15 @@ const CategoryUnknown Category = 0
 // number of categories that can be registered.
 const CategoryMax = 30
 
+// shardPadding pads each shard to 64 bytes so they don't share a cache line.
+const shardPadding = 64 - unsafe.Sizeof(CategoryStatsShard{})
+
+// paddedCategoryStatsShard is a single shard of a category's statistics.
+type paddedCategoryStatsShard struct {
+	CategoryStatsShard
+	_ [shardPadding]byte
+}
+
 func (c Category) String() string {
 	return categories[c].name
 }
@@ -169,20 +179,32 @@ type CategoryStatsAggregate struct {
 	CategoryStats CategoryStats
 }
 
-const numCategoryStatsShards = 16
+// numCategoryStatsShards must be a power of 2. We initialize it to GOMAXPROCS
+// (rounded up to the nearest power of 2) or 16, whichever is larger.
+var numCategoryStatsShards = func() int {
+	p := runtime.GOMAXPROCS(0)
+	n := 16
+	for n < p {
+		n *= 2
+	}
+	return n
+}()
 
-type categoryStatsWithMu struct {
-	mu sync.Mutex
-	// Protected by mu.
-	stats CategoryStats
+// CategoryStatsShard holds CategoryStats with a mutex
+// to ensure safe access.
+type CategoryStatsShard struct {
+	mu struct {
+		sync.Mutex
+		stats CategoryStats
+	}
 }
 
 // Accumulate implements the IterStatsAccumulator interface.
-func (c *categoryStatsWithMu) Accumulate(
+func (c *CategoryStatsShard) Accumulate(
 	blockBytes, blockBytesInCache uint64, blockReadDuration time.Duration,
 ) {
 	c.mu.Lock()
-	c.stats.aggregate(blockBytes, blockBytesInCache, blockReadDuration)
+	c.mu.stats.aggregate(blockBytes, blockBytesInCache, blockReadDuration)
 	c.mu.Unlock()
 }
 
@@ -199,11 +221,7 @@ type CategoryStatsCollector struct {
 // contention on the category stats mutex has been observed.
 type shardedCategoryStats struct {
 	Category Category
-	shards   [numCategoryStatsShards]struct {
-		categoryStatsWithMu
-		// Pad each shard to 64 bytes so they don't share a cache line.
-		_ [64 - unsafe.Sizeof(categoryStatsWithMu{})]byte
-	}
+	shards   []paddedCategoryStatsShard
 }
 
 // getStats retrieves the aggregated stats for the category, summing across all
@@ -214,7 +232,7 @@ func (s *shardedCategoryStats) getStats() CategoryStatsAggregate {
 	}
 	for i := range s.shards {
 		s.shards[i].mu.Lock()
-		agg.CategoryStats.aggregate(s.shards[i].stats.BlockBytes, s.shards[i].stats.BlockBytesInCache, s.shards[i].stats.BlockReadDuration)
+		agg.CategoryStats.aggregate(s.shards[i].mu.stats.BlockBytes, s.shards[i].mu.stats.BlockBytesInCache, s.shards[i].mu.stats.BlockReadDuration)
 		s.shards[i].mu.Unlock()
 	}
 	return agg
@@ -222,20 +240,21 @@ func (s *shardedCategoryStats) getStats() CategoryStatsAggregate {
 
 // Accumulator returns a stats accumulator for the given category. The provided
 // p is used to detrmine which shard to write stats to.
-func (c *CategoryStatsCollector) Accumulator(p uint64, category Category) IterStatsAccumulator {
+func (c *CategoryStatsCollector) Accumulator(p uint64, category Category) *CategoryStatsShard {
 	v, ok := c.statsMap.Load(category)
 	if !ok {
 		c.mu.Lock()
 		v, _ = c.statsMap.LoadOrStore(category, &shardedCategoryStats{
 			Category: category,
+			shards:   make([]paddedCategoryStatsShard, numCategoryStatsShards),
 		})
 		c.mu.Unlock()
 	}
 	s := v.(*shardedCategoryStats)
 	// This equation is taken from:
 	// https://en.wikipedia.org/wiki/Linear_congruential_generator#Parameters_in_common_use
-	shard := ((p * 25214903917) >> 32) & (numCategoryStatsShards - 1)
-	return &s.shards[shard].categoryStatsWithMu
+	shard := ((p * 25214903917) >> 32) & uint64(numCategoryStatsShards-1)
+	return &s.shards[shard].CategoryStatsShard
 }
 
 // GetStats returns the aggregated stats.
@@ -251,8 +270,3 @@ func (c *CategoryStatsCollector) GetStats() []CategoryStatsAggregate {
 	})
 	return stats
 }
-
-type IterStatsAccumulator interface {
-	// Accumulate accumulates the provided stats.
-	Accumulate(blockBytes, blockBytesInCache uint64, blockReadDuration time.Duration)
-}
diff --git a/sstable/block/compression.go b/sstable/block/compression.go
index 15b079b4..4ab6a643 100644
--- a/sstable/block/compression.go
+++ b/sstable/block/compression.go
@@ -212,18 +212,6 @@ func (b PhysicalBlock) Clone() PhysicalBlock {
 	return PhysicalBlock{data: data, trailer: b.trailer}
 }
 
-// CloneUsingBuf makes a copy of the block data, using the given slice if it has
-// enough capacity.
-func (b PhysicalBlock) CloneUsingBuf(buf []byte) (_ PhysicalBlock, newBuf []byte) {
-	newBuf = append(buf[:0], b.data...)
-	return PhysicalBlock{data: newBuf, trailer: b.trailer}, newBuf
-}
-
-// IsCompressed returns true if the block is compressed.
-func (b *PhysicalBlock) IsCompressed() bool {
-	return CompressionIndicator(b.trailer[0]) != NoCompressionIndicator
-}
-
 // WriteTo writes the block (including its trailer) to the provided Writable. If
 // err == nil, n is the number of bytes successfully written to the Writable.
 //
@@ -247,35 +235,37 @@ func (b *PhysicalBlock) WriteTo(w objstorage.Writable) (n int, err error) {
 }
 
 // CompressAndChecksum compresses and checksums the provided block, returning
-// the compressed block and its trailer. The dst argument is used for the
-// compressed payload if it's sufficiently large. If it's not, a new buffer is
-// allocated and *dst is updated to point to it.
+// the compressed block and its trailer. The result is appended to the dst
+// argument.
 //
 // If the compressed block is not sufficiently smaller than the original block,
-// the compressed payload is discarded and the original, uncompressed block is
-// used to avoid unnecessary decompression overhead at read time.
+// the compressed payload is discarded and the original, uncompressed block data
+// is used to avoid unnecessary decompression overhead at read time.
 func CompressAndChecksum(
-	dst *[]byte, block []byte, compression Compression, checksummer *Checksummer,
+	dst *[]byte, blockData []byte, compression Compression, checksummer *Checksummer,
 ) PhysicalBlock {
+	buf := (*dst)[:0]
 	// Compress the buffer, discarding the result if the improvement isn't at
 	// least 12.5%.
 	algo := NoCompressionIndicator
 	if compression != NoCompression {
-		var compressed []byte
-		algo, compressed = compress(compression, block, *dst)
-		if algo != NoCompressionIndicator && cap(compressed) > cap(*dst) {
-			*dst = compressed[:cap(compressed)]
-		}
-		if len(compressed) < len(block)-len(block)/8 {
-			block = compressed
-		} else {
+		algo, buf = compress(compression, blockData, buf)
+		if len(buf) >= len(blockData)-len(blockData)/8 {
 			algo = NoCompressionIndicator
 		}
 	}
+	if algo == NoCompressionIndicator {
+		// We don't want to use the given blockData buffer directly: typically the
+		// result will be written to disk and that can mangle the buffer, leading to
+		// fragile code.
+		buf = append(buf[:0], blockData...)
+	}
+
+	*dst = buf
 
 	// Calculate the checksum.
-	pb := PhysicalBlock{data: block}
-	checksum := checksummer.Checksum(block, byte(algo))
+	pb := PhysicalBlock{data: buf}
+	checksum := checksummer.Checksum(buf, byte(algo))
 	pb.trailer = MakeTrailer(byte(algo), checksum)
 	return pb
 }
@@ -375,19 +365,8 @@ func (b *Buffer) CompressAndChecksum() (PhysicalBlock, *BufHandle) {
 	// Grab a buffer to use as the destination for compression.
 	compressedBuf := compressedBuffers.Get()
 	pb := CompressAndChecksum(&compressedBuf.b, b.h.b, b.compression, &b.checksummer)
-	if pb.IsCompressed() {
-		// Compression was fruitful, and pb's data points into compressedBuf. We
-		// can reuse b.Buffer because we've copied the compressed data.
-		b.h.b = b.h.b[:0]
-		return pb, compressedBuf
-	}
-	// Compression was not fruitful, and pb's data points into b.h. The
-	// compressedBuf we retrieved from the pool isn't needed, but our b.h is.
-	// Use the compressedBuf as the new b.h.
-	pbHandle := b.h
-	b.h = compressedBuf
 	b.h.b = b.h.b[:0]
-	return pb, pbHandle
+	return pb, compressedBuf
 }
 
 // SetCompression changes the compression algorithm used by CompressAndChecksum.
diff --git a/sstable/block/compression_cgo.go b/sstable/block/compression_cgo.go
index 76f6b66b..4ef0bda7 100644
--- a/sstable/block/compression_cgo.go
+++ b/sstable/block/compression_cgo.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build cgo
-// +build cgo
 
 package block
 
diff --git a/sstable/block/compression_nocgo.go b/sstable/block/compression_nocgo.go
index 0c787a05..4d34aa46 100644
--- a/sstable/block/compression_nocgo.go
+++ b/sstable/block/compression_nocgo.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build !cgo
-// +build !cgo
 
 package block
 
diff --git a/sstable/colblk/prefix_bytes.go b/sstable/colblk/prefix_bytes.go
index ee9c749b..91c3ff07 100644
--- a/sstable/colblk/prefix_bytes.go
+++ b/sstable/colblk/prefix_bytes.go
@@ -662,9 +662,10 @@ type PrefixBytesBuilder struct {
 	// may remain very small while the physical in-memory size of the
 	// in-progress data slice may grow very large. This may pose memory usage
 	// problems during block building.
-	data       []byte // The raw, concatenated keys w/o any prefix compression
-	nKeys      int    // The number of keys added to the builder
-	bundleSize int    // The number of keys per bundle
+	data               []byte // The raw, concatenated keys w/o any prefix compression
+	nKeys              int    // The number of keys added to the builder
+	bundleSize         int    // The number of keys per bundle
+	completedBundleLen int    // The encoded size of completed bundles
 	// sizings maintains metadata about the size of the accumulated data at both
 	// nKeys and nKeys-1. Information for the state after the most recently
 	// added key is stored at (b.nKeys+1)%2.
@@ -754,7 +755,6 @@ type prefixBytesSizing struct {
 	// INVARIANT: currentBundlePrefixLen >= blockPrefixLen
 	currentBundlePrefixLen    int          // the length of the "current" bundle's prefix
 	currentBundlePrefixOffset int          // the index of the offset of the "current" bundle's prefix
-	completedBundleLen        int          // the encoded size of completed bundles
 	compressedDataLen         int          // the compressed, encoded size of data
 	offsetEncoding            UintEncoding // the encoding necessary to encode the offsets
 }
@@ -763,10 +763,10 @@ func (sz *prefixBytesSizing) String() string {
 	return fmt.Sprintf("lastKeyOff:%d offsetCount:%d blockPrefixLen:%d\n"+
 		"currentBundleDistinct{Len,Keys}: (%d,%d)\n"+
 		"currentBundlePrefix{Len,Offset}: (%d,%d)\n"+
-		"completedBundleLen:%d compressedDataLen:%d offsetEncoding:%s",
+		"compressedDataLen:%d offsetEncoding:%s",
 		sz.lastKeyOff, sz.offsetCount, sz.blockPrefixLen, sz.currentBundleDistinctLen,
 		sz.currentBundleDistinctKeys, sz.currentBundlePrefixLen, sz.currentBundlePrefixOffset,
-		sz.completedBundleLen, sz.compressedDataLen, sz.offsetEncoding)
+		sz.compressedDataLen, sz.offsetEncoding)
 }
 
 // Put adds the provided key to the column. The provided key must be
@@ -781,7 +781,7 @@ func (sz *prefixBytesSizing) String() string {
 func (b *PrefixBytesBuilder) Put(key []byte, bytesSharedWithPrev int) {
 	currIdx := b.nKeys & 1 // %2
 	curr := &b.sizings[currIdx]
-	prev := &b.sizings[1-currIdx]
+	prev := &b.sizings[currIdx^1]
 
 	if invariants.Enabled {
 		if len(key) == 0 {
@@ -801,29 +801,30 @@ func (b *PrefixBytesBuilder) Put(key []byte, bytesSharedWithPrev int) {
 		}
 	}
 
-	switch {
-	case b.nKeys == 0:
-		// We're adding the first key to the block.
-		// Set a placeholder offset for the block prefix length.
-		b.addOffset(0)
-		// Set a placeholder offset for the bundle prefix length.
-		b.addOffset(0)
-		b.nKeys++
-		b.data = append(b.data, key...)
-		b.addOffset(uint32(len(b.data)))
-		*curr = prefixBytesSizing{
-			lastKeyOff:                0,
-			offsetCount:               b.offsets.count,
-			blockPrefixLen:            min(len(key), int(b.maxShared)),
-			currentBundleDistinctLen:  len(key),
-			currentBundleDistinctKeys: 1,
-			currentBundlePrefixLen:    min(len(key), int(b.maxShared)),
-			currentBundlePrefixOffset: 1,
-			completedBundleLen:        0,
-			compressedDataLen:         len(key),
-			offsetEncoding:            DetermineUintEncoding(0, uint64(len(key)), UintEncodingRowThreshold),
+	// Check if this is the first key in a bundle.
+	if b.nKeys&(b.bundleSize-1) == 0 {
+		if b.nKeys == 0 {
+			// We're adding the first key to the block.
+			// Set a placeholder offset for the block prefix length.
+			b.addOffset(0)
+			// Set a placeholder offset for the bundle prefix length.
+			b.addOffset(0)
+			b.nKeys++
+			b.data = append(b.data, key...)
+			b.addOffset(uint32(len(b.data)))
+			*curr = prefixBytesSizing{
+				lastKeyOff:                0,
+				offsetCount:               b.offsets.count,
+				blockPrefixLen:            min(len(key), int(b.maxShared)),
+				currentBundleDistinctLen:  len(key),
+				currentBundleDistinctKeys: 1,
+				currentBundlePrefixLen:    min(len(key), int(b.maxShared)),
+				currentBundlePrefixOffset: 1,
+				compressedDataLen:         len(key),
+				offsetEncoding:            DetermineUintEncodingNoDelta(uint64(len(key))),
+			}
+			return
 		}
-	case b.nKeys&(b.bundleSize-1) == 0:
 		// We're starting a new bundle.
 
 		// Set the bundle prefix length of the previous bundle.
@@ -832,7 +833,7 @@ func (b *PrefixBytesBuilder) Put(key []byte, bytesSharedWithPrev int) {
 
 		// Finalize the encoded size of the previous bundle.
 		bundleSizeJustCompleted := prev.currentBundleDistinctLen - (prev.currentBundleDistinctKeys-1)*prev.currentBundlePrefixLen
-		completedBundleSize := prev.completedBundleLen + bundleSizeJustCompleted
+		b.completedBundleLen += bundleSizeJustCompleted
 
 		// Update the block prefix length if necessary. The caller tells us how
 		// many bytes of prefix this key shares with the previous key. The block
@@ -842,64 +843,62 @@ func (b *PrefixBytesBuilder) Put(key []byte, bytesSharedWithPrev int) {
 		blockPrefixLen := min(prev.blockPrefixLen, bytesSharedWithPrev)
 		b.nKeys++
 		*curr = prefixBytesSizing{
-			lastKeyOff:         len(b.data),
-			offsetCount:        b.offsets.count + 2,
-			blockPrefixLen:     blockPrefixLen,
-			completedBundleLen: completedBundleSize,
+			lastKeyOff:     len(b.data),
+			offsetCount:    b.offsets.count + 2,
+			blockPrefixLen: blockPrefixLen,
 			// We're adding the first key to the current bundle. Initialize
 			// the current bundle prefix.
 			currentBundlePrefixOffset: b.offsets.count,
 			currentBundlePrefixLen:    min(len(key), int(b.maxShared)),
 			currentBundleDistinctLen:  len(key),
 			currentBundleDistinctKeys: 1,
-			compressedDataLen:         completedBundleSize + len(key) - (b.bundleCount(b.nKeys)-1)*blockPrefixLen,
+			compressedDataLen:         b.completedBundleLen + len(key) - (b.bundleCount(b.nKeys)-1)*blockPrefixLen,
 		}
-		curr.offsetEncoding = DetermineUintEncoding(0, uint64(curr.compressedDataLen), UintEncodingRowThreshold)
+		curr.offsetEncoding = DetermineUintEncodingNoDelta(uint64(curr.compressedDataLen))
 		b.data = append(b.data, key...)
 		b.addOffset(0) // Placeholder for bundle prefix.
 		b.addOffset(uint32(len(b.data)))
-	default:
-		// Adding a new key to an existing bundle.
-		b.nKeys++
-
-		if bytesSharedWithPrev == len(key) {
-			// Duplicate key; don't add it to the data slice and don't adjust
-			// currentBundleDistinct{Len,Keys}.
-			*curr = *prev
-			curr.offsetCount++
-			b.addOffset(b.offsets.elems.At(b.offsets.count - 1))
-			return
-		}
+		return
+	}
+	// We're adding a new key to an existing bundle.
+	b.nKeys++
+
+	if bytesSharedWithPrev == len(key) {
+		// Duplicate key; don't add it to the data slice and don't adjust
+		// currentBundleDistinct{Len,Keys}.
+		*curr = *prev
+		curr.offsetCount++
+		b.addOffset(b.offsets.elems.At(b.offsets.count - 1))
+		return
+	}
 
-		// Update the bundle prefix length. Note that the shared prefix length
-		// can only shrink as new values are added. During construction, the
-		// bundle prefix value is stored contiguously in the data array so even
-		// if the bundle prefix length changes no adjustment is needed to that
-		// value or to the first key in the bundle.
-		*curr = prefixBytesSizing{
-			lastKeyOff:                len(b.data),
-			offsetCount:               prev.offsetCount + 1,
-			blockPrefixLen:            min(prev.blockPrefixLen, bytesSharedWithPrev),
-			currentBundleDistinctLen:  prev.currentBundleDistinctLen + len(key),
-			currentBundleDistinctKeys: prev.currentBundleDistinctKeys + 1,
-			currentBundlePrefixLen:    min(prev.currentBundlePrefixLen, bytesSharedWithPrev),
-			currentBundlePrefixOffset: prev.currentBundlePrefixOffset,
-			completedBundleLen:        prev.completedBundleLen,
-		}
-		// Compute the correct compressedDataLen.
-		curr.compressedDataLen = curr.completedBundleLen +
-			curr.currentBundleDistinctLen -
-			(curr.currentBundleDistinctKeys-1)*curr.currentBundlePrefixLen
-		// Currently compressedDataLen is correct, except that it includes the block
-		// prefix length for all bundle prefixes. Adjust the length to account for
-		// the block prefix being stripped from every bundle except the first one.
-		curr.compressedDataLen -= (b.bundleCount(b.nKeys) - 1) * curr.blockPrefixLen
-		// The compressedDataLen is the largest offset we'll need to encode in the
-		// offset table.
-		curr.offsetEncoding = DetermineUintEncoding(0, uint64(curr.compressedDataLen), UintEncodingRowThreshold)
-		b.data = append(b.data, key...)
-		b.addOffset(uint32(len(b.data)))
+	// Update the bundle prefix length. Note that the shared prefix length
+	// can only shrink as new values are added. During construction, the
+	// bundle prefix value is stored contiguously in the data array so even
+	// if the bundle prefix length changes no adjustment is needed to that
+	// value or to the first key in the bundle.
+	*curr = prefixBytesSizing{
+		lastKeyOff:                len(b.data),
+		offsetCount:               prev.offsetCount + 1,
+		blockPrefixLen:            min(prev.blockPrefixLen, bytesSharedWithPrev),
+		currentBundleDistinctLen:  prev.currentBundleDistinctLen + len(key),
+		currentBundleDistinctKeys: prev.currentBundleDistinctKeys + 1,
+		currentBundlePrefixLen:    min(prev.currentBundlePrefixLen, bytesSharedWithPrev),
+		currentBundlePrefixOffset: prev.currentBundlePrefixOffset,
 	}
+	// Compute the correct compressedDataLen.
+	curr.compressedDataLen = b.completedBundleLen +
+		curr.currentBundleDistinctLen -
+		(curr.currentBundleDistinctKeys-1)*curr.currentBundlePrefixLen
+	// Currently compressedDataLen is correct, except that it includes the block
+	// prefix length for all bundle prefixes. Adjust the length to account for
+	// the block prefix being stripped from every bundle except the first one.
+	curr.compressedDataLen -= (b.bundleCount(b.nKeys) - 1) * curr.blockPrefixLen
+	// The compressedDataLen is the largest offset we'll need to encode in the
+	// offset table.
+	curr.offsetEncoding = DetermineUintEncodingNoDelta(uint64(curr.compressedDataLen))
+	b.data = append(b.data, key...)
+	b.addOffset(uint32(len(b.data)))
 }
 
 // UnsafeGet returns the zero-indexed i'th key added to the builder through Put.
@@ -913,7 +912,7 @@ func (b *PrefixBytesBuilder) UnsafeGet(i int) []byte {
 		lastKeyOff := b.sizings[i&1].lastKeyOff
 		return b.data[lastKeyOff:]
 	case b.nKeys - 2:
-		lastKeyOff := b.sizings[(i+1)&1].lastKeyOff
+		lastKeyOff := b.sizings[(i^1)&1].lastKeyOff
 		secondLastKeyOff := b.sizings[i&1].lastKeyOff
 		if secondLastKeyOff == lastKeyOff {
 			// The last key is a duplicate of the second-to-last key.
@@ -1050,7 +1049,7 @@ func (b *PrefixBytesBuilder) Finish(
 	buf[offset] = byte(b.bundleShift)
 	offset++
 
-	sz := &b.sizings[(rows+1)%2]
+	sz := &b.sizings[rows&1^1]
 	stringDataOffset := uintColumnSize(uint32(sz.offsetCount), offset, sz.offsetEncoding)
 	if sz.offsetEncoding.IsDelta() {
 		panic(errors.AssertionFailedf("offsets never need delta encoding"))
@@ -1087,7 +1086,7 @@ func (b *PrefixBytesBuilder) Size(rows int, offset uint32) uint32 {
 	} else if rows != b.nKeys && rows != b.nKeys-1 {
 		panic(errors.AssertionFailedf("PrefixBytes has accumulated %d keys, asked to Size %d", b.nKeys, rows))
 	}
-	sz := &b.sizings[(rows+1)%2]
+	sz := &b.sizings[rows&1^1]
 	// The 1-byte bundleSize.
 	offset++
 	// Compute the size of the offsets table.
diff --git a/sstable/colblk/uints.go b/sstable/colblk/uints.go
index 2613c70c..cf462519 100644
--- a/sstable/colblk/uints.go
+++ b/sstable/colblk/uints.go
@@ -110,16 +110,34 @@ func DetermineUintEncoding(minValue, maxValue uint64, numRows int) UintEncoding
 	return makeUintEncoding(b, isDelta)
 }
 
+// DetermineUintEncodingNoDelta is a more efficient variant of
+// DetermineUintEncoding when minValue is zero (or we don't need a delta
+// encoding).
+func DetermineUintEncodingNoDelta(maxValue uint64) UintEncoding {
+	return makeUintEncoding(byteWidth(maxValue), false /* isDelta */)
+}
+
+// byteWidthTable maps a number’s bit‐length to the number of bytes needed.
+var byteWidthTable = [65]uint8{
+	// 0 bits => 0 bytes
+	0,
+	// 1..8 bits => 1 byte
+	1, 1, 1, 1, 1, 1, 1, 1,
+	// 9..16 bits => 2 bytes
+	2, 2, 2, 2, 2, 2, 2, 2,
+	// 17..32 bits => 4 bytes
+	4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
+	// 33..64 bits => 8 bytes
+	8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
+	8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
+}
+
 // byteWidth returns the number of bytes necessary to represent the given value,
 // either 0, 1, 2, 4, or 8.
 func byteWidth(maxValue uint64) uint8 {
-	// b the number of bytes necessary to represent maxValue.
-	b := (uint8(bits.Len64(maxValue)) + 7) >> 3
-	// Round up to the nearest power of 2 by subtracting one and filling out all bits.
-	b--
-	b |= b >> 1
-	b |= b >> 2
-	return b + 1
+	// bits.Len64 returns 0 for 0, 1..64 for others.
+	// We then simply return the precomputed result.
+	return byteWidthTable[bits.Len64(maxValue)]
 }
 
 func makeUintEncoding(width uint8, isDelta bool) UintEncoding {
diff --git a/sstable/colblk_writer.go b/sstable/colblk_writer.go
index 35fa1b9a..9a63bc85 100644
--- a/sstable/colblk_writer.go
+++ b/sstable/colblk_writer.go
@@ -667,14 +667,6 @@ func (w *RawColumnWriter) enqueueDataBlock(
 		w.opts.Compression,
 		&cb.blockBuf.checksummer,
 	)
-	if !cb.physical.IsCompressed() {
-		// If the block isn't compressed, cb.physical's underlying data points
-		// directly into a buffer owned by w.dataBlock. Clone it before passing
-		// it to the write queue to be asynchronously written to disk.
-		// TODO(jackson): Should we try to avoid this clone by tracking the
-		// lifetime of the DataBlockWriters?
-		cb.physical, cb.blockBuf.dataBuf = cb.physical.CloneUsingBuf(cb.blockBuf.dataBuf)
-	}
 	return w.enqueuePhysicalBlock(cb, separator)
 }
 
@@ -1164,14 +1156,6 @@ func (w *RawColumnWriter) addDataBlock(b, sep []byte, bhp block.HandleWithProper
 		w.opts.Compression,
 		&cb.blockBuf.checksummer,
 	)
-	if !cb.physical.IsCompressed() {
-		// If the block isn't compressed, cb.physical's underlying data points
-		// directly into a buffer owned by w.dataBlock. Clone it before passing
-		// it to the write queue to be asynchronously written to disk.
-		// TODO(jackson): Should we try to avoid this clone by tracking the
-		// lifetime of the DataBlockWriters?
-		cb.physical, cb.blockBuf.dataBuf = cb.physical.CloneUsingBuf(cb.blockBuf.dataBuf)
-	}
 	if err := w.enqueuePhysicalBlock(cb, sep); err != nil {
 		return err
 	}
diff --git a/sstable/layout.go b/sstable/layout.go
index 6c8c0003..c16f5787 100644
--- a/sstable/layout.go
+++ b/sstable/layout.go
@@ -18,7 +18,6 @@ import (
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/binfmt"
 	"github.com/cockroachdb/pebble/internal/bytealloc"
-	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/sstableinternal"
 	"github.com/cockroachdb/pebble/internal/treeprinter"
 	"github.com/cockroachdb/pebble/objstorage"
@@ -675,8 +674,7 @@ func (w *layoutWriter) Abort() {
 }
 
 // WriteDataBlock constructs a trailer for the provided data block and writes
-// the block and trailer to the writer. It returns the block's handle. It can
-// mangle b.
+// the block and trailer to the writer. It returns the block's handle.
 func (w *layoutWriter) WriteDataBlock(b []byte, buf *blockBuf) (block.Handle, error) {
 	return w.writeBlock(b, w.compression, buf)
 }
@@ -692,8 +690,6 @@ func (w *layoutWriter) WritePrecompressedDataBlock(blk block.PhysicalBlock) (blo
 // second-level) and writes the block and trailer to the writer. It remembers
 // the last-written index block's handle and adds it to the file's meta index
 // when the writer is finished.
-//
-// WriteIndexBlock can mangle b.
 func (w *layoutWriter) WriteIndexBlock(b []byte) (block.Handle, error) {
 	h, err := w.writeBlock(b, w.compression, &w.buf)
 	if err == nil {
@@ -716,8 +712,6 @@ func (w *layoutWriter) WriteFilterBlock(f filterWriter) (bh block.Handle, err er
 // WritePropertiesBlock constructs a trailer for the provided properties block
 // and writes the block and trailer to the writer. It automatically adds the
 // properties block to the file's meta index when the writer is finished.
-//
-// WritePropertiesBlock can mangle b.
 func (w *layoutWriter) WritePropertiesBlock(b []byte) (block.Handle, error) {
 	return w.writeNamedBlock(b, metaPropertiesName)
 }
@@ -725,8 +719,6 @@ func (w *layoutWriter) WritePropertiesBlock(b []byte) (block.Handle, error) {
 // WriteRangeKeyBlock constructs a trailer for the provided range key block and
 // writes the block and trailer to the writer. It automatically adds the range
 // key block to the file's meta index when the writer is finished.
-//
-// WriteRangeKeyBlock can mangle the block data.
 func (w *layoutWriter) WriteRangeKeyBlock(b []byte) (block.Handle, error) {
 	return w.writeNamedBlock(b, metaRangeKeyName)
 }
@@ -735,13 +727,10 @@ func (w *layoutWriter) WriteRangeKeyBlock(b []byte) (block.Handle, error) {
 // block and writes the block and trailer to the writer. It automatically adds
 // the range deletion block to the file's meta index when the writer is
 // finished.
-//
-// WriteRangeDeletionBlock can mangle the block data.
 func (w *layoutWriter) WriteRangeDeletionBlock(b []byte) (block.Handle, error) {
 	return w.writeNamedBlock(b, metaRangeDelV2Name)
 }
 
-// writeNamedBlock can mangle the block data.
 func (w *layoutWriter) writeNamedBlock(b []byte, name string) (bh block.Handle, err error) {
 	bh, err = w.writeBlock(b, block.NoCompression, &w.buf)
 	if err == nil {
@@ -770,20 +759,12 @@ func (w *layoutWriter) WriteValueIndexBlock(
 	return h, nil
 }
 
-// writeBlock checksums, compresses, and writes out a block. It can mangle b.
+// writeBlock checksums, compresses, and writes out a block.
 func (w *layoutWriter) writeBlock(
 	b []byte, compression block.Compression, buf *blockBuf,
 ) (block.Handle, error) {
 	pb := block.CompressAndChecksum(&buf.dataBuf, b, compression, &buf.checksummer)
 	h, err := w.writePrecompressedBlock(pb)
-	// This method is allowed to mangle b, but that only happens when the block
-	// data is not compressible. Mangle it anyway in invariant builds to catch
-	// callers that don't handle this.
-	if invariants.Enabled && invariants.Sometimes(1) {
-		for i := range b {
-			b[i] = 0xFF
-		}
-	}
 	return h, err
 }
 
@@ -820,10 +801,10 @@ func (w *layoutWriter) Write(blockWithTrailer []byte) (n int, err error) {
 // clearFromCache removes the block at the provided offset from the cache. This provides defense in
 // depth against bugs which cause cache collisions.
 func (w *layoutWriter) clearFromCache(offset uint64) {
-	if w.cacheOpts.Cache != nil {
+	if w.cacheOpts.CacheHandle != nil {
 		// TODO(peter): Alternatively, we could add the uncompressed value to the
 		// cache.
-		w.cacheOpts.Cache.Delete(w.cacheOpts.CacheID, w.cacheOpts.FileNum, offset)
+		w.cacheOpts.CacheHandle.Delete(w.cacheOpts.FileNum, offset)
 	}
 }
 
diff --git a/sstable/reader_iter.go b/sstable/reader_iter.go
index f15f4bef..e479c38f 100644
--- a/sstable/reader_iter.go
+++ b/sstable/reader_iter.go
@@ -49,7 +49,12 @@ type Iterator interface {
 	// NextPrefix implements (base.InternalIterator).NextPrefix.
 	NextPrefix(succKey []byte) *base.InternalKV
 
-	SetCloseHook(fn func(i Iterator) error)
+	// SetCloseHook sets a function that will be called when the iterator is
+	// closed.  This is used by the file cache to release the reference count on
+	// the open sstable.Reader when the iterator is closed. The closures takes
+	// the iterator as a parameter to enable invariant-build tracking of leaked
+	// iterators.
+	SetCloseHook(fn func(any))
 }
 
 // Iterator positioning optimizations and singleLevelIterator and
diff --git a/sstable/reader_iter_single_lvl.go b/sstable/reader_iter_single_lvl.go
index 559becd3..ba378a1f 100644
--- a/sstable/reader_iter_single_lvl.go
+++ b/sstable/reader_iter_single_lvl.go
@@ -71,7 +71,7 @@ type singleLevelIterator[I any, PI indexBlockIterator[I], D any, PD dataBlockIte
 	vbRH         objstorage.ReadHandle
 	vbRHPrealloc objstorageprovider.PreallocatedReadHandle
 	err          error
-	closeHook    func(i Iterator) error
+	closeHook    func(i any)
 
 	readBlockEnv block.ReadEnv
 
@@ -1513,9 +1513,11 @@ func (i *singleLevelIterator[I, PI, D, PD]) Error() error {
 	return i.err
 }
 
-// SetCloseHook sets a function that will be called when the iterator is
-// closed.
-func (i *singleLevelIterator[I, PI, D, PD]) SetCloseHook(fn func(i Iterator) error) {
+// SetCloseHook sets a function that will be called when the iterator is closed.
+// This is used by the file cache to release the reference count on the open
+// sstable.Reader when the iterator is closed. The closures takes the iterator
+// as a parameter to enable invariant-build tracking of leaked iterators.
+func (i *singleLevelIterator[I, PI, D, PD]) SetCloseHook(fn func(i any)) {
 	i.closeHook = fn
 }
 
@@ -1543,10 +1545,10 @@ func (i *singleLevelIterator[I, PI, D, PD]) closeInternal() error {
 		panic("Close called on interator in pool")
 	}
 
-	var err error
 	if i.closeHook != nil {
-		err = firstError(err, i.closeHook(i))
+		i.closeHook(i)
 	}
+	var err error
 	err = firstError(err, PD(&i.data).Close())
 	err = firstError(err, PI(&i.index).Close())
 	if i.indexFilterRH != nil {
diff --git a/sstable/reader_iter_two_lvl.go b/sstable/reader_iter_two_lvl.go
index 437d12eb..83c010fc 100644
--- a/sstable/reader_iter_two_lvl.go
+++ b/sstable/reader_iter_two_lvl.go
@@ -1015,7 +1015,7 @@ func (i *twoLevelIterator[I, PI, D, PD]) SetContext(ctx context.Context) {
 	i.secondLevel.SetContext(ctx)
 }
 
-func (i *twoLevelIterator[I, PI, D, PD]) SetCloseHook(fn func(i Iterator) error) {
+func (i *twoLevelIterator[I, PI, D, PD]) SetCloseHook(fn func(i any)) {
 	i.secondLevel.SetCloseHook(fn)
 }
 
diff --git a/sstable/rowblk/rowblk_iter.go b/sstable/rowblk/rowblk_iter.go
index cdb46964..bc744f98 100644
--- a/sstable/rowblk/rowblk_iter.go
+++ b/sstable/rowblk/rowblk_iter.go
@@ -1561,21 +1561,11 @@ start:
 	return &i.ikv
 }
 
-// Key returns the internal key at the current iterator position.
-func (i *Iter) Key() *base.InternalKey {
-	return &i.ikv.K
-}
-
 // KV returns the internal KV at the current iterator position.
 func (i *Iter) KV() *base.InternalKV {
 	return &i.ikv
 }
 
-// Value returns the value at the current iterator position.
-func (i *Iter) Value() base.LazyValue {
-	return i.ikv.V
-}
-
 // Error implements internalIterator.Error, as documented in the pebble
 // package.
 func (i *Iter) Error() error {
@@ -1766,8 +1756,8 @@ func (i *RawIter) cacheEntry() {
 	i.cachedBuf = append(i.cachedBuf, i.key...)
 }
 
-// SeekGE implements internalIterator.SeekGE, as documented in the pebble
-// package.
+// SeekGE repositions the iterator to point to the first KV with a user key
+// greater than or equal to key. It returns whether the iterator is valid.
 func (i *RawIter) SeekGE(key []byte) bool {
 	// Find the index of the smallest restart point whose key is > the key
 	// sought; index will be numRestarts if there is no such restart point.
@@ -1802,15 +1792,16 @@ func (i *RawIter) SeekGE(key []byte) bool {
 	return i.Valid()
 }
 
-// First implements internalIterator.First, as documented in the pebble
-// package.
+// First repositions the iterator so that it is pointing to the first key in the
+// block, returning whether or not the iterator is still valid.
 func (i *RawIter) First() bool {
 	i.offset = 0
 	i.loadEntry()
 	return i.Valid()
 }
 
-// Last implements internalIterator.Last, as documented in the pebble package.
+// Last repositions the iterator so that it is pointing to the last key in the
+// block, returning whether or not the iterator is still valid.
 func (i *RawIter) Last() bool {
 	// Seek forward from the last restart point.
 	i.offset = offsetInBlock(binary.LittleEndian.Uint32(i.data[i.restarts+4*offsetInBlock(i.numRestarts-1):]))
@@ -1829,8 +1820,8 @@ func (i *RawIter) Last() bool {
 	return i.Valid()
 }
 
-// Next implements internalIterator.Next, as documented in the pebble
-// package.
+// Next repositions the iterator one step forward, returning whether or not the
+// iterator is still valid.
 func (i *RawIter) Next() bool {
 	i.offset = i.nextOffset
 	if !i.Valid() {
@@ -1840,8 +1831,8 @@ func (i *RawIter) Next() bool {
 	return true
 }
 
-// Prev implements internalIterator.Prev, as documented in the pebble
-// package.
+// Prev repositions the iterator one step backwards, returning whether or not
+// the iterator is still valid.
 func (i *RawIter) Prev() bool {
 	if n := len(i.cached) - 1; n > 0 && i.cached[n].offset == i.offset {
 		i.nextOffset = i.offset
@@ -1883,31 +1874,24 @@ func (i *RawIter) Prev() bool {
 	return true
 }
 
-// Key implements internalIterator.Key, as documented in the pebble package.
+// Key returns the key at the current iterator position. The iterator must be
+// Valid().
 func (i *RawIter) Key() base.InternalKey {
 	return i.ikey
 }
 
-// Value implements internalIterator.Value, as documented in the pebble
-// package.
+// Value returns the value at the current iterator position. The iterator must
+// be Valid().
 func (i *RawIter) Value() []byte {
 	return i.val
 }
 
-// Valid implements internalIterator.Valid, as documented in the pebble
-// package.
+// Valid returns true if the iterator is currently positioned at a valid KV.
 func (i *RawIter) Valid() bool {
 	return i.offset >= 0 && i.offset < i.restarts
 }
 
-// Error implements internalIterator.Error, as documented in the pebble
-// package.
-func (i *RawIter) Error() error {
-	return nil
-}
-
-// Close implements internalIterator.Close, as documented in the pebble
-// package.
+// Close implements closes the iterator.
 func (i *RawIter) Close() error {
 	i.val = nil
 	return nil
diff --git a/sstable/rowblk/rowblk_rewrite.go b/sstable/rowblk/rowblk_rewrite.go
index d1667ee1..48b64cbc 100644
--- a/sstable/rowblk/rowblk_rewrite.go
+++ b/sstable/rowblk/rowblk_rewrite.go
@@ -10,6 +10,7 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/bytealloc"
+	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/sstable/block"
 )
 
@@ -78,6 +79,9 @@ func (r *Rewriter) RewriteSuffixes(
 		// in the block, which includes the 1-byte prefix. This is fine since bw
 		// also does not know about the prefix and will preserve it in bw.add.
 		v := kv.InPlaceValue()
+		if invariants.Enabled && invariants.Sometimes(10) {
+			r.comparer.ValidateKey.MustValidate(r.scratchKey.UserKey)
+		}
 		r.writer.Add(r.scratchKey, v)
 		if start.UserKey == nil {
 			// Copy the first key.
diff --git a/sstable/rowblk_writer.go b/sstable/rowblk_writer.go
index 4ce0a9ef..97d237dd 100644
--- a/sstable/rowblk_writer.go
+++ b/sstable/rowblk_writer.go
@@ -51,6 +51,7 @@ type RawRowWriter struct {
 	compression          block.Compression
 	separator            Separator
 	successor            Successor
+	validateKey          base.ValidateKey
 	tableFormat          TableFormat
 	isStrictObsolete     bool
 	writingToLowestLevel bool
@@ -1186,6 +1187,12 @@ func (w *RawRowWriter) indexEntrySep(
 	} else {
 		sep = prevKey.Separator(w.compare, w.separator, dataBlockBuf.sepScratch[:0], key)
 	}
+	if invariants.Enabled && invariants.Sometimes(25) {
+		if w.compare(prevKey.UserKey, sep.UserKey) > 0 {
+			panic(errors.AssertionFailedf("prevKey.UserKey > sep.UserKey: %s > %s",
+				prevKey.UserKey, sep.UserKey))
+		}
+	}
 	return sep
 }
 
@@ -1214,6 +1221,11 @@ func (w *RawRowWriter) addIndexEntry(
 		// In particular, it must have a non-zero length.
 		return nil
 	}
+	if invariants.Enabled {
+		if err := w.validateKey.Validate(sep.UserKey); err != nil {
+			return err
+		}
+	}
 
 	encoded := bhp.EncodeVarints(tmp)
 	if flushIndexBuf != nil {
@@ -1532,6 +1544,8 @@ func (w *RawRowWriter) Close() (err error) {
 			return err
 		}
 		prevKey := w.dataBlockBuf.dataBlock.CurKey()
+		// NB: prevKey.UserKey may be nil if there are no keys and we're forcing
+		// the creation of an empty data block.
 		if err := w.addIndexEntrySync(prevKey, InternalKey{}, bhp, w.dataBlockBuf.tmp[:]); err != nil {
 			return err
 		}
@@ -1713,6 +1727,7 @@ func newRowWriter(writable objstorage.Writable, o WriterOptions) *RawRowWriter {
 		compression:                o.Compression,
 		separator:                  o.Comparer.Separator,
 		successor:                  o.Comparer.Successor,
+		validateKey:                o.Comparer.ValidateKey,
 		tableFormat:                o.TableFormat,
 		isStrictObsolete:           o.IsStrictObsolete,
 		writingToLowestLevel:       o.WritingToLowestLevel,
@@ -1953,13 +1968,9 @@ func (w *RawRowWriter) addDataBlock(b, sep []byte, bhp block.HandleWithPropertie
 		w.layout.compression,
 		&blockBuf.checksummer,
 	)
-	if !pb.IsCompressed() {
-		// If the block isn't compressed, pb's underlying data points
-		// directly b. Clone it before writing it, as writing can mangle the buffer.
-		pb, blockBuf.dataBuf = pb.CloneUsingBuf(blockBuf.dataBuf)
-	}
 
 	// layout.WriteDataBlock keeps layout.offset up-to-date for us.
+	// Note that this can mangle the pb data.
 	bh, err := w.layout.writePrecompressedBlock(pb)
 	if err != nil {
 		return err
diff --git a/sstable/suffix_rewriter.go b/sstable/suffix_rewriter.go
index 8387f189..67b0f34f 100644
--- a/sstable/suffix_rewriter.go
+++ b/sstable/suffix_rewriter.go
@@ -178,6 +178,12 @@ func rewriteDataBlocksInParallel(
 					if err != nil {
 						return err
 					}
+					if err := r.Comparer.ValidateKey.Validate(output[i].start.UserKey); err != nil {
+						return err
+					}
+					if err := r.Comparer.ValidateKey.Validate(output[i].end.UserKey); err != nil {
+						return err
+					}
 					compressedBuf = compressedBuf[:cap(compressedBuf)]
 					finished := block.CompressAndChecksum(&compressedBuf, outputBlock, opts.Compression, &checksummer)
 					output[i].physical = finished.CloneWithByteAlloc(&blockAlloc)
@@ -219,6 +225,12 @@ func rewriteRangeKeyBlockToWriter(r *Reader, w RawWriter, from, to []byte) error
 		if !s.Valid() {
 			break
 		}
+		if err := r.Comparer.ValidateKey.Validate(s.Start); err != nil {
+			return err
+		}
+		if err := r.Comparer.ValidateKey.Validate(s.End); err != nil {
+			return err
+		}
 		for i := range s.Keys {
 			if s.Keys[i].Kind() != base.InternalKeyKindRangeKeySet {
 				return errBadKind
@@ -328,6 +340,9 @@ func RewriteKeySuffixesViaWriter(
 		if err != nil {
 			return nil, err
 		}
+		if invariants.Enabled && invariants.Sometimes(10) {
+			r.Comparer.ValidateKey.MustValidate(scratch.UserKey)
+		}
 		if err := w.AddWithForceObsolete(scratch, val, false); err != nil {
 			return nil, err
 		}
diff --git a/table_stats.go b/table_stats.go
index 89b2f17d..469d394a 100644
--- a/table_stats.go
+++ b/table_stats.go
@@ -249,7 +249,7 @@ func (d *DB) scanReadStateTableStats(
 			// to have a mismatch; the size stored in the FileBacking is just the part
 			// of the file that is referenced by this Pebble instance, not the size of
 			// the whole object.
-			objMeta, err := d.objProvider.Lookup(fileTypeTable, f.FileBacking.DiskFileNum)
+			objMeta, err := d.objProvider.Lookup(base.FileTypeTable, f.FileBacking.DiskFileNum)
 			if err != nil {
 				// Set `moreRemain` so we'll try again.
 				moreRemain = true
diff --git a/tool/db.go b/tool/db.go
index b7664435..76f8bf53 100644
--- a/tool/db.go
+++ b/tool/db.go
@@ -377,8 +377,8 @@ func (d *dbT) openDBInternal(dir string, openOptions ...OpenOption) (*pebble.DB,
 	for _, opt := range d.openOptions {
 		opt.Apply(dir, &opts)
 	}
-	opts.Cache = pebble.NewCache(128 << 20 /* 128 MB */)
-	defer opts.Cache.Unref()
+	opts.Cache = nil
+	opts.CacheSize = 128 * 1024 * 1024
 	return pebble.Open(dir, &opts)
 }
 
@@ -609,7 +609,8 @@ func (d *dbT) runExcise(cmd *cobra.Command, args []string) {
 		return
 	}
 
-	dbOpts := d.opts.EnsureDefaults()
+	d.opts.EnsureDefaults()
+	dbOpts := d.opts
 	// Disable all processes that would try to open tables: table stats,
 	// consistency check, automatic compactions.
 	dbOpts.DisableTableStats = true
diff --git a/tool/find.go b/tool/find.go
index 85574471..89cf0d86 100644
--- a/tool/find.go
+++ b/tool/find.go
@@ -410,6 +410,8 @@ func (f *findT) searchLogs(stdout io.Writer, searchKey []byte, refs []findRef) [
 func (f *findT) searchTables(stdout io.Writer, searchKey []byte, refs []findRef) []findRef {
 	cache := pebble.NewCache(128 << 20 /* 128 MB */)
 	defer cache.Unref()
+	ch := cache.NewHandle()
+	defer ch.Close()
 
 	f.tableRefs = make(map[base.FileNum]bool)
 	for _, fl := range f.tables {
@@ -443,8 +445,8 @@ func (f *findT) searchTables(stdout io.Writer, searchKey []byte, refs []findRef)
 			opts.Comparers = f.comparers
 			opts.Mergers = f.mergers
 			opts.CacheOpts = sstableinternal.CacheOptions{
-				Cache:   cache,
-				FileNum: fl.DiskFileNum,
+				CacheHandle: ch,
+				FileNum:     fl.DiskFileNum,
 			}
 			readable, err := sstable.NewSimpleReadable(tf)
 			if err != nil {
diff --git a/tool/make_incorrect_manifests.go b/tool/make_incorrect_manifests.go
index 90aa87d0..f0ae2030 100644
--- a/tool/make_incorrect_manifests.go
+++ b/tool/make_incorrect_manifests.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build make_incorrect_manifests
-// +build make_incorrect_manifests
 
 // Run using: go run -tags make_incorrect_manifests ./tool/make_incorrect_manifests.go
 package main
diff --git a/tool/sstable.go b/tool/sstable.go
index fa99824e..71b3897a 100644
--- a/tool/sstable.go
+++ b/tool/sstable.go
@@ -17,6 +17,7 @@ import (
 
 	"github.com/cockroachdb/pebble"
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/cache"
 	"github.com/cockroachdb/pebble/internal/humanize"
 	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/rangedel"
@@ -143,7 +144,7 @@ inclusive-inclusive range specified by --start and --end.
 	return s
 }
 
-func (s *sstableT) newReader(f vfs.File) (*sstable.Reader, error) {
+func (s *sstableT) newReader(f vfs.File, cacheHandle *cache.Handle) (*sstable.Reader, error) {
 	readable, err := sstable.NewSimpleReadable(f)
 	if err != nil {
 		return nil, err
@@ -151,31 +152,14 @@ func (s *sstableT) newReader(f vfs.File) (*sstable.Reader, error) {
 	o := s.opts.MakeReaderOptions()
 	o.Comparers = s.comparers
 	o.Mergers = s.mergers
-	c := pebble.NewCache(128 << 20 /* 128 MB */)
-	defer c.Unref()
-	o.CacheOpts = sstableinternal.CacheOptions{Cache: c}
+	o.CacheOpts = sstableinternal.CacheOptions{CacheHandle: cacheHandle}
 	return sstable.NewReader(context.Background(), readable, o)
 }
 
 func (s *sstableT) runCheck(cmd *cobra.Command, args []string) {
 	stdout, stderr := cmd.OutOrStdout(), cmd.OutOrStderr()
-	s.foreachSstable(stderr, args, func(arg string) {
-		f, err := s.opts.FS.Open(arg)
-		if err != nil {
-			fmt.Fprintf(stderr, "%s\n", err)
-			return
-		}
-
-		fmt.Fprintf(stdout, "%s\n", arg)
-
-		r, err := s.newReader(f)
-
-		if err != nil {
-			fmt.Fprintf(stdout, "%s\n", err)
-			return
-		}
-		defer r.Close()
-
+	s.foreachSstable(stderr, args, func(path string, r *sstable.Reader) {
+		fmt.Fprintf(stdout, "%s\n", path)
 		// Update the internal formatter if this comparator has one specified.
 		s.fmtKey.setForComparer(r.Properties.ComparerName, s.comparers)
 		s.fmtValue.setForComparer(r.Properties.ComparerName, s.comparers)
@@ -227,21 +211,8 @@ func (s *sstableT) runCheck(cmd *cobra.Command, args []string) {
 
 func (s *sstableT) runLayout(cmd *cobra.Command, args []string) {
 	stdout, stderr := cmd.OutOrStdout(), cmd.OutOrStderr()
-	s.foreachSstable(stderr, args, func(arg string) {
-		f, err := s.opts.FS.Open(arg)
-		if err != nil {
-			fmt.Fprintf(stderr, "%s\n", err)
-			return
-		}
-
-		fmt.Fprintf(stdout, "%s\n", arg)
-
-		r, err := s.newReader(f)
-		if err != nil {
-			fmt.Fprintf(stdout, "%s\n", err)
-			return
-		}
-		defer r.Close()
+	s.foreachSstable(stderr, args, func(path string, r *sstable.Reader) {
+		fmt.Fprintf(stdout, "%s\n", path)
 
 		// Update the internal formatter if this comparator has one specified.
 		s.fmtKey.setForComparer(r.Properties.ComparerName, s.comparers)
@@ -267,28 +238,15 @@ func (s *sstableT) runLayout(cmd *cobra.Command, args []string) {
 
 func (s *sstableT) runProperties(cmd *cobra.Command, args []string) {
 	stdout, stderr := cmd.OutOrStdout(), cmd.OutOrStderr()
-	s.foreachSstable(stderr, args, func(arg string) {
-		f, err := s.opts.FS.Open(arg)
-		if err != nil {
-			fmt.Fprintf(stderr, "%s\n", err)
-			return
-		}
-
-		fmt.Fprintf(stdout, "%s\n", arg)
-
-		r, err := s.newReader(f)
-		if err != nil {
-			fmt.Fprintf(stdout, "%s\n", err)
-			return
-		}
-		defer r.Close()
+	s.foreachSstable(stderr, args, func(path string, r *sstable.Reader) {
+		fmt.Fprintf(stdout, "%s\n", path)
 
 		if s.verbose {
 			fmt.Fprintf(stdout, "%s", r.Properties.String())
 			return
 		}
 
-		stat, err := f.Stat()
+		stat, err := s.opts.FS.Stat(path)
 		if err != nil {
 			fmt.Fprintf(stderr, "%s\n", err)
 			return
@@ -360,32 +318,19 @@ func (s *sstableT) runProperties(cmd *cobra.Command, args []string) {
 
 func (s *sstableT) runScan(cmd *cobra.Command, args []string) {
 	stdout, stderr := cmd.OutOrStdout(), cmd.OutOrStderr()
-	s.foreachSstable(stderr, args, func(arg string) {
-		f, err := s.opts.FS.Open(arg)
-		if err != nil {
-			fmt.Fprintf(stderr, "%s\n", err)
-			return
-		}
+	s.foreachSstable(stderr, args, func(path string, r *sstable.Reader) {
+		// Update the internal formatter if this comparator has one specified.
+		s.fmtKey.setForComparer(r.Properties.ComparerName, s.comparers)
+		s.fmtValue.setForComparer(r.Properties.ComparerName, s.comparers)
 
 		// In filter-mode, we prefix ever line that is output with the sstable
 		// filename.
 		var prefix string
 		if s.filter == nil {
-			fmt.Fprintf(stdout, "%s\n", arg)
+			fmt.Fprintf(stdout, "%s\n", path)
 		} else {
-			prefix = fmt.Sprintf("%s: ", arg)
-		}
-
-		r, err := s.newReader(f)
-		if err != nil {
-			fmt.Fprintf(stdout, "%s%s\n", prefix, err)
-			return
+			prefix = fmt.Sprintf("%s: ", path)
 		}
-		defer r.Close()
-
-		// Update the internal formatter if this comparator has one specified.
-		s.fmtKey.setForComparer(r.Properties.ComparerName, s.comparers)
-		s.fmtValue.setForComparer(r.Properties.ComparerName, s.comparers)
 
 		iter, err := r.NewIter(sstable.NoTransforms, nil, s.end)
 		if err != nil {
@@ -553,41 +498,48 @@ func (s *sstableT) runScan(cmd *cobra.Command, args []string) {
 
 func (s *sstableT) runSpace(cmd *cobra.Command, args []string) {
 	stdout, stderr := cmd.OutOrStdout(), cmd.OutOrStderr()
-	s.foreachSstable(stderr, args, func(arg string) {
-		f, err := s.opts.FS.Open(arg)
+	s.foreachSstable(stderr, args, func(path string, r *sstable.Reader) {
+		bytes, err := r.EstimateDiskUsage(s.start, s.end)
 		if err != nil {
 			fmt.Fprintf(stderr, "%s\n", err)
 			return
 		}
-		r, err := s.newReader(f)
+		fmt.Fprintf(stdout, "%s: %d\n", path, bytes)
+	})
+}
+
+// foreachSstable opens each sstable specified in the args (if an arg is a
+// directory, it is walked for sstable files) and calls the given function.
+func (s *sstableT) foreachSstable(
+	stderr io.Writer, args []string, fn func(path string, r *sstable.Reader),
+) {
+	pathFn := func(path string) {
+		f, err := s.opts.FS.Open(path)
 		if err != nil {
 			fmt.Fprintf(stderr, "%s\n", err)
 			return
 		}
-		defer r.Close()
 
-		bytes, err := r.EstimateDiskUsage(s.start, s.end)
+		c := pebble.NewCache(128 << 20 /* 128 MB */)
+		defer c.Unref()
+		ch := c.NewHandle()
+		defer ch.Close()
+
+		r, err := s.newReader(f, ch)
 		if err != nil {
-			fmt.Fprintf(stderr, "%s\n", err)
+			fmt.Fprintf(stderr, "%s: %s\n", path, err)
 			return
 		}
-		fmt.Fprintf(stdout, "%s: %d\n", arg, bytes)
-	})
-}
+		defer r.Close()
+		fn(path, r)
+	}
 
-func (s *sstableT) foreachSstable(stderr io.Writer, args []string, fn func(arg string)) {
-	// Loop over args, invoking fn for each file. Each directory is recursively
 	// listed and fn is invoked on any file with an .sst or .ldb suffix.
 	for _, arg := range args {
-		info, err := s.opts.FS.Stat(arg)
-		if err != nil || !info.IsDir() {
-			fn(arg)
-			continue
-		}
 		walk(stderr, s.opts.FS, arg, func(path string) {
 			switch filepath.Ext(path) {
 			case ".sst", ".ldb":
-				fn(path)
+				pathFn(path)
 			}
 		})
 	}
diff --git a/tool/util.go b/tool/util.go
index 6747a1bf..59fc0b16 100644
--- a/tool/util.go
+++ b/tool/util.go
@@ -302,24 +302,25 @@ func formatSpan(w io.Writer, fmtKey keyFormatter, fmtValue valueFormatter, s *ke
 	}
 }
 
-func walk(stderr io.Writer, fs vfs.FS, dir string, fn func(path string)) {
-	paths, err := fs.List(dir)
+// walk calls fn for each file in dir and its subdirectories (or just on the
+// path if it is not a directory),
+func walk(stderr io.Writer, fs vfs.FS, path string, fn func(path string)) {
+	info, err := fs.Stat(path)
 	if err != nil {
-		fmt.Fprintf(stderr, "%s: %v\n", dir, err)
+		fmt.Fprintf(stderr, "%s: %v\n", path, err)
+		return
+	}
+	if !info.IsDir() {
+		fn(path)
+		return
+	}
+	paths, err := fs.List(path)
+	if err != nil {
+		fmt.Fprintf(stderr, "%s: %v\n", path, err)
 		return
 	}
 	sort.Strings(paths)
 	for _, part := range paths {
-		path := fs.PathJoin(dir, part)
-		info, err := fs.Stat(path)
-		if err != nil {
-			fmt.Fprintf(stderr, "%s: %v\n", path, err)
-			continue
-		}
-		if info.IsDir() {
-			walk(stderr, fs, path, fn)
-		} else {
-			fn(path)
-		}
+		walk(stderr, fs, fs.PathJoin(path, part), fn)
 	}
 }
diff --git a/version_set.go b/version_set.go
index 6e121a66..b8dc4f2a 100644
--- a/version_set.go
+++ b/version_set.go
@@ -78,13 +78,14 @@ type versionSet struct {
 	// A pointer to versionSet.addObsoleteLocked. Avoids allocating a new closure
 	// on the creation of every version.
 	obsoleteFn        func(obsolete []*fileBacking)
-	obsoleteTables    []tableInfo
+	obsoleteTables    []objectInfo
+	obsoleteBlobs     []objectInfo
 	obsoleteManifests []fileInfo
 	obsoleteOptions   []fileInfo
 
 	// Zombie tables which have been removed from the current version but are
 	// still referenced by an inuse iterator.
-	zombieTables map[base.DiskFileNum]tableInfo
+	zombieTables map[base.DiskFileNum]objectInfo
 
 	// virtualBackings contains information about the FileBackings which support
 	// virtual sstables in the latest version. It is mainly used to determine when
@@ -129,7 +130,9 @@ type versionSet struct {
 	rotationHelper record.RotationHelper
 }
 
-type tableInfo struct {
+// objectInfo describes an object in object storage (either a sstable or a blob
+// file).
+type objectInfo struct {
 	fileInfo
 	isLocal bool
 }
@@ -152,7 +155,7 @@ func (vs *versionSet) init(
 	vs.dynamicBaseLevel = true
 	vs.versions.Init(mu)
 	vs.obsoleteFn = vs.addObsoleteLocked
-	vs.zombieTables = make(map[base.DiskFileNum]tableInfo)
+	vs.zombieTables = make(map[base.DiskFileNum]objectInfo)
 	vs.virtualBackings = manifest.MakeVirtualBackings()
 	vs.nextFileNum.Store(1)
 	vs.manifestMarker = marker
@@ -194,14 +197,14 @@ func (vs *versionSet) create(
 	}
 	if err == nil {
 		// NB: Move() is responsible for syncing the data directory.
-		if err = vs.manifestMarker.Move(base.MakeFilename(fileTypeManifest, vs.manifestFileNum)); err != nil {
+		if err = vs.manifestMarker.Move(base.MakeFilename(base.FileTypeManifest, vs.manifestFileNum)); err != nil {
 			vs.opts.Logger.Fatalf("MANIFEST set current failed: %v", err)
 		}
 	}
 
 	vs.opts.EventListener.ManifestCreated(ManifestCreateInfo{
 		JobID:   int(jobID),
-		Path:    base.MakeFilepath(vs.fs, vs.dirname, fileTypeManifest, vs.manifestFileNum),
+		Path:    base.MakeFilepath(vs.fs, vs.dirname, base.FileTypeManifest, vs.manifestFileNum),
 		FileNum: vs.manifestFileNum,
 		Err:     err,
 	})
@@ -224,7 +227,7 @@ func (vs *versionSet) load(
 	vs.init(dirname, provider, opts, marker, getFormatMajorVersion, mu)
 
 	vs.manifestFileNum = manifestFileNum
-	manifestPath := base.MakeFilepath(opts.FS, dirname, fileTypeManifest, vs.manifestFileNum)
+	manifestPath := base.MakeFilepath(opts.FS, dirname, base.FileTypeManifest, vs.manifestFileNum)
 	manifestFilename := opts.FS.PathBase(manifestPath)
 
 	// Read the versionEdits in the manifest file.
@@ -571,7 +574,7 @@ func (vs *versionSet) logAndApply(
 			if err := vs.createManifest(vs.dirname, newManifestFileNum, minUnflushedLogNum, nextFileNum, newManifestVirtualBackings); err != nil {
 				vs.opts.EventListener.ManifestCreated(ManifestCreateInfo{
 					JobID:   int(jobID),
-					Path:    base.MakeFilepath(vs.fs, vs.dirname, fileTypeManifest, newManifestFileNum),
+					Path:    base.MakeFilepath(vs.fs, vs.dirname, base.FileTypeManifest, newManifestFileNum),
 					FileNum: newManifestFileNum,
 					Err:     err,
 				})
@@ -600,12 +603,12 @@ func (vs *versionSet) logAndApply(
 		}
 		if newManifestFileNum != 0 {
 			// NB: Move() is responsible for syncing the data directory.
-			if err := vs.manifestMarker.Move(base.MakeFilename(fileTypeManifest, newManifestFileNum)); err != nil {
+			if err := vs.manifestMarker.Move(base.MakeFilename(base.FileTypeManifest, newManifestFileNum)); err != nil {
 				return errors.Wrap(err, "MANIFEST set current failed")
 			}
 			vs.opts.EventListener.ManifestCreated(ManifestCreateInfo{
 				JobID:   int(jobID),
-				Path:    base.MakeFilepath(vs.fs, vs.dirname, fileTypeManifest, newManifestFileNum),
+				Path:    base.MakeFilepath(vs.fs, vs.dirname, base.FileTypeManifest, newManifestFileNum),
 				FileNum: newManifestFileNum,
 			})
 		}
@@ -635,7 +638,7 @@ func (vs *versionSet) logAndApply(
 	// will unref the previous version which could result in addObsoleteLocked
 	// being called.
 	for _, b := range zombieBackings {
-		vs.zombieTables[b.backing.DiskFileNum] = tableInfo{
+		vs.zombieTables[b.backing.DiskFileNum] = objectInfo{
 			fileInfo: fileInfo{
 				FileNum:  b.backing.DiskFileNum,
 				FileSize: b.backing.Size,
@@ -879,7 +882,7 @@ func (vs *versionSet) createManifest(
 	virtualBackings []*fileBacking,
 ) (err error) {
 	var (
-		filename     = base.MakeFilepath(vs.fs, dirname, fileTypeManifest, fileNum)
+		filename     = base.MakeFilepath(vs.fs, dirname, base.FileTypeManifest, fileNum)
 		manifestFile vfs.File
 		manifest     *record.Writer
 	)
@@ -1035,7 +1038,7 @@ func (vs *versionSet) addObsoleteLocked(obsolete []*fileBacking) {
 		return
 	}
 
-	obsoleteFileInfo := make([]tableInfo, len(obsolete))
+	obsoleteFileInfo := make([]objectInfo, len(obsolete))
 	for i, bs := range obsolete {
 		obsoleteFileInfo[i].FileNum = bs.DiskFileNum
 		obsoleteFileInfo[i].FileSize = bs.Size
diff --git a/vfs/default_linux.go b/vfs/default_linux.go
index b505ce0c..2b4894e9 100644
--- a/vfs/default_linux.go
+++ b/vfs/default_linux.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build linux
-// +build linux
 
 package vfs
 
diff --git a/vfs/default_unix.go b/vfs/default_unix.go
index 34a4a3e7..593d9d06 100644
--- a/vfs/default_unix.go
+++ b/vfs/default_unix.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build darwin || dragonfly || freebsd || netbsd || openbsd || solaris
-// +build darwin dragonfly freebsd netbsd openbsd solaris
 
 package vfs
 
diff --git a/vfs/default_windows.go b/vfs/default_windows.go
index 05777f20..9882df14 100644
--- a/vfs/default_windows.go
+++ b/vfs/default_windows.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build windows
-// +build windows
 
 package vfs
 
diff --git a/vfs/disk_usage_linux.go b/vfs/disk_usage_linux.go
index 4f0bcd9d..c967a66f 100644
--- a/vfs/disk_usage_linux.go
+++ b/vfs/disk_usage_linux.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build linux
-// +build linux
 
 package vfs
 
diff --git a/vfs/disk_usage_netbsd.go b/vfs/disk_usage_netbsd.go
index 8b7e9dd9..88398781 100644
--- a/vfs/disk_usage_netbsd.go
+++ b/vfs/disk_usage_netbsd.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build netbsd
-// +build netbsd
 
 package vfs
 
diff --git a/vfs/disk_usage_openbsd.go b/vfs/disk_usage_openbsd.go
index 19e9c5cf..536ad1ef 100644
--- a/vfs/disk_usage_openbsd.go
+++ b/vfs/disk_usage_openbsd.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build openbsd
-// +build openbsd
 
 package vfs
 
diff --git a/vfs/disk_usage_unix.go b/vfs/disk_usage_unix.go
index 95d9a7c3..a025f746 100644
--- a/vfs/disk_usage_unix.go
+++ b/vfs/disk_usage_unix.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build darwin || dragonfly || freebsd
-// +build darwin dragonfly freebsd
 
 package vfs
 
diff --git a/vfs/disk_usage_windows.go b/vfs/disk_usage_windows.go
index a380bcbf..600877f2 100644
--- a/vfs/disk_usage_windows.go
+++ b/vfs/disk_usage_windows.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build windows
-// +build windows
 
 package vfs
 
diff --git a/vfs/errors_unix.go b/vfs/errors_unix.go
index 2d05e14b..eea5f80f 100644
--- a/vfs/errors_unix.go
+++ b/vfs/errors_unix.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build darwin || dragonfly || freebsd || linux || openbsd || netbsd
-// +build darwin dragonfly freebsd linux openbsd netbsd
 
 package vfs
 
diff --git a/vfs/errors_windows.go b/vfs/errors_windows.go
index 65920fc7..4ba99824 100644
--- a/vfs/errors_windows.go
+++ b/vfs/errors_windows.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build windows
-// +build windows
 
 package vfs
 
diff --git a/vfs/fadvise_generic.go b/vfs/fadvise_generic.go
index d6b9b41a..b92cb5f4 100644
--- a/vfs/fadvise_generic.go
+++ b/vfs/fadvise_generic.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build !linux
-// +build !linux
 
 package vfs
 
diff --git a/vfs/fadvise_linux.go b/vfs/fadvise_linux.go
index 6bb4db13..5aa75a22 100644
--- a/vfs/fadvise_linux.go
+++ b/vfs/fadvise_linux.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build linux
-// +build linux
 
 package vfs
 
diff --git a/vfs/file_lock_generic.go b/vfs/file_lock_generic.go
index 6177da69..1b936536 100644
--- a/vfs/file_lock_generic.go
+++ b/vfs/file_lock_generic.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !solaris && !windows
-// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!windows
 
 package vfs
 
diff --git a/vfs/file_lock_unix.go b/vfs/file_lock_unix.go
index 4d05e8ce..4f93e3cc 100644
--- a/vfs/file_lock_unix.go
+++ b/vfs/file_lock_unix.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris
-// +build darwin dragonfly freebsd linux netbsd openbsd solaris
 
 package vfs
 
diff --git a/vfs/file_lock_windows.go b/vfs/file_lock_windows.go
index 31fd17a8..d0237081 100644
--- a/vfs/file_lock_windows.go
+++ b/vfs/file_lock_windows.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build windows
-// +build windows
 
 package vfs
 



Test patch:
diff --git a/cmd/pebble/replay_test.go b/cmd/pebble/replay_test.go
index b3f62255..5c4f729a 100644
--- a/cmd/pebble/replay_test.go
+++ b/cmd/pebble/replay_test.go
@@ -9,7 +9,6 @@ import (
 	"testing"
 
 	"github.com/cockroachdb/pebble"
-	"github.com/cockroachdb/pebble/internal/cache"
 	"github.com/stretchr/testify/require"
 )
 
@@ -30,14 +29,14 @@ func TestParseOptionsStr(t *testing.T) {
 		},
 		{
 			c:       replayConfig{optionsString: fmt.Sprintf(`[Options] cache_size=%d`, 16<<20 /* 16MB */)},
-			options: &pebble.Options{Cache: cache.New(16 << 20 /* 16 MB */)},
+			options: &pebble.Options{CacheSize: 16 * 1024 * 1024},
 		},
 		{
 			c: replayConfig{
 				maxCacheSize:  16 << 20, /* 16 MB */
 				optionsString: fmt.Sprintf(`[Options] cache_size=%d`, int64(10<<30 /* 10 GB */)),
 			},
-			options: &pebble.Options{Cache: cache.New(16 << 20 /* 16 MB */)},
+			options: &pebble.Options{CacheSize: 16 * 1024 * 1024},
 		},
 		{
 			c: replayConfig{optionsString: `[Options] [Level "0"] target_file_size=222`},
@@ -59,19 +58,18 @@ func TestParseOptionsStr(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		o := new(pebble.Options)
-		require.NoError(t, tc.c.parseCustomOptions(tc.c.optionsString, o))
-		o.EnsureDefaults()
-		got := o.String()
+		t.Run("", func(t *testing.T) {
+			o := new(pebble.Options)
+			require.NoError(t, tc.c.parseCustomOptions(tc.c.optionsString, o))
+			o.EnsureDefaults()
+			got := o.String()
 
-		tc.options.EnsureDefaults()
-		want := tc.options.String()
-		require.Equal(t, want, got)
-		if o.Cache != nil {
-			o.Cache.Unref()
-		}
-		if tc.options.Cache != nil {
-			tc.options.Cache.Unref()
-		}
+			tc.options.EnsureDefaults()
+			want := tc.options.String()
+			require.Equal(t, want, got)
+			if o.Cache != nil {
+				o.Cache.Unref()
+			}
+		})
 	}
 }
diff --git a/cockroachkvs/cockroachkvs_bench_test.go b/cockroachkvs/cockroachkvs_bench_test.go
index e3fdb4f3..a7baf1bc 100644
--- a/cockroachkvs/cockroachkvs_bench_test.go
+++ b/cockroachkvs/cockroachkvs_bench_test.go
@@ -98,13 +98,14 @@ func benchmarkRandSeekInSST(
 	// overhead.
 	c := cache.New(obj.Size() * 2)
 	defer c.Unref()
+	ch := c.NewHandle()
+	defer ch.Close()
 	ctx := context.Background()
 	readerOpts := sstable.ReaderOptions{
 		ReaderOptions: block.ReaderOptions{
 			CacheOpts: sstableinternal.CacheOptions{
-				Cache:   c,
-				CacheID: c.NewID(),
-				FileNum: 1,
+				CacheHandle: ch,
+				FileNum:     1,
 			},
 		},
 		Comparer:   writerOpts.Comparer,
diff --git a/cockroachkvs/cockroachkvs_test.go b/cockroachkvs/cockroachkvs_test.go
index 7227a8ea..1c4cb860 100644
--- a/cockroachkvs/cockroachkvs_test.go
+++ b/cockroachkvs/cockroachkvs_test.go
@@ -79,6 +79,46 @@ func TestComparer(t *testing.T) {
 	}
 }
 
+func TestComparerFuncs(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+
+	var buf bytes.Buffer
+	datadriven.RunTest(t, "testdata/comparer_funcs",
+		func(t *testing.T, td *datadriven.TestData) string {
+			buf.Reset()
+			switch td.Cmd {
+			case "separator":
+				var keys [][]byte
+				var dst []byte
+				for _, line := range crstrings.Lines(td.Input) {
+					keys = keys[:0]
+					for _, formattedKey := range strings.Fields(line) {
+						k := ParseFormattedKey(formattedKey)
+						keys = append(keys, k)
+					}
+					if len(keys) != 2 {
+						t.Fatalf("expected two keys, but found %d", len(keys))
+					}
+					dst = Comparer.Separator(dst[:0], keys[0], keys[1])
+					fmt.Fprintf(&buf, "Separator(%q [%x], %q [%x]) = %q [%x]\n",
+						FormatKey(keys[0]), keys[0], FormatKey(keys[1]), keys[1], FormatKey(dst), dst)
+				}
+				return buf.String()
+			case "successor":
+				var dst []byte
+				for _, line := range crstrings.Lines(td.Input) {
+					k := ParseFormattedKey(line)
+					dst = Comparer.Successor(dst[:0], k)
+					fmt.Fprintf(&buf, "Successor(%q [%x]) = %q [%x]\n",
+						FormatKey(k), k, FormatKey(dst), dst)
+				}
+				return buf.String()
+			default:
+				panic(fmt.Sprintf("unknown command: %s", td.Cmd))
+			}
+		})
+}
+
 func TestKeySchema_KeyWriter(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 
@@ -633,7 +673,7 @@ func parseUserKey(userKeyStr string) []byte {
 		}
 	}
 	k := append(append([]byte(roachKey), 0), parseVersion(versionStr)...)
-	checkEngineKey(k)
+	validateEngineKey.MustValidate(k)
 	return k
 }
 
@@ -657,7 +697,7 @@ func parseVersion(versionStr string) []byte {
 		if len(ret) != 14 && len(ret) != 10 {
 			panic(fmt.Sprintf("expected 10 or 14-length ret got %d", len(ret)))
 		}
-		checkEngineKey(ret)
+		validateEngineKey.MustValidate(ret)
 		// TODO(jackson): Refactor to allow us to generate a suffix without a
 		// sentinel byte rather than stripping it like this.
 		return ret[1:]
diff --git a/cockroachkvs/key_schema_test.go b/cockroachkvs/key_schema_test.go
index c3309bce..dcaae8f3 100644
--- a/cockroachkvs/key_schema_test.go
+++ b/cockroachkvs/key_schema_test.go
@@ -161,7 +161,7 @@ func TestKeySchema_RandomKeys(t *testing.T) {
 		if n := len(kv.K.UserKey); n > len(keys[k]) {
 			t.Fatalf("key %q is longer than original key %q", kv.K.UserKey, keys[k])
 		}
-		checkEngineKey(kv.K.UserKey)
+		validateEngineKey.MustValidate(kv.K.UserKey)
 
 		// We write keys[k] as the value too, so check that it's verbatim equal.
 		value, callerOwned, err := kv.V.Value(valBuf)
diff --git a/cockroachkvs/testdata/comparer_funcs b/cockroachkvs/testdata/comparer_funcs
new file mode 100644
index 00000000..6034fc44
--- /dev/null
+++ b/cockroachkvs/testdata/comparer_funcs
@@ -0,0 +1,41 @@
+successor
+foo
+bar
+foo@1000000,0
+foo@1000000.000051212,0
+foo@1000000,8
+foo@1000000.000051212,7
+foo@01,cf379e85-6371-42e6-acc8-ed5259fa177a
+----
+Successor(foo [666f6f00]) = g [6700]
+Successor(bar [62617200]) = c [6300]
+Successor(foo@1000000,0 [666f6f0000038d7ea4c6800009]) = g [6700]
+Successor(foo@1000000.000051212,0 [666f6f0000038d7ea4c7480c09]) = g [6700]
+Successor(foo@1000000,8 [666f6f0000038d7ea4c68000000000080d]) = g [6700]
+Successor(foo@1000000.000051212,7 [666f6f0000038d7ea4c7480c000000070d]) = g [6700]
+Successor(foo@01,cf379e85-6371-42e6-acc8-ed5259fa177a [666f6f0001cf379e85637142e6acc8ed5259fa177a12]) = g [6700]
+
+# The successor for an empty string should be a key consisting only of the 0x00
+# byte.
+successor
+hex:
+----
+Successor(<empty> []) =  [00]
+
+separator
+bar foo
+foo@1000000,0 foo@1000000,0
+foo@1000000,1 foo@1000000,0
+foo@01,a65a118c-5dcb-452b-bd82-7dd17e121138 foo@01,cf379e85-6371-42e6-acc8-ed5259fa177a
+fax@01,a65a118c-5dcb-452b-bd82-7dd17e121138 foo@01,cf379e85-6371-42e6-acc8-ed5259fa177a
+----
+Separator(bar [62617200], foo [666f6f00]) = c [6300]
+Separator(foo@1000000,0 [666f6f0000038d7ea4c6800009], foo@1000000,0 [666f6f0000038d7ea4c6800009]) = foo@1000000,0 [666f6f0000038d7ea4c6800009]
+Separator(foo@1000000,1 [666f6f0000038d7ea4c68000000000010d], foo@1000000,0 [666f6f0000038d7ea4c6800009]) = foo@1000000,1 [666f6f0000038d7ea4c68000000000010d]
+Separator(foo@01,a65a118c-5dcb-452b-bd82-7dd17e121138 [666f6f0001a65a118c5dcb452bbd827dd17e12113812], foo@01,cf379e85-6371-42e6-acc8-ed5259fa177a [666f6f0001cf379e85637142e6acc8ed5259fa177a12]) = foo@01,a65a118c-5dcb-452b-bd82-7dd17e121138 [666f6f0001a65a118c5dcb452bbd827dd17e12113812]
+Separator(fax@01,a65a118c-5dcb-452b-bd82-7dd17e121138 [6661780001a65a118c5dcb452bbd827dd17e12113812], foo@01,cf379e85-6371-42e6-acc8-ed5259fa177a [666f6f0001cf379e85637142e6acc8ed5259fa177a12]) = fb [666200]
+
+separator
+@1,0 a@2,0
+----
+Separator(@1,0 [00000000003b9aca0009], a@2,0 [6100000000007735940009]) = @1,0 [00000000003b9aca0009]
diff --git a/cockroachkvs/testdata/key_schema_key_seeker b/cockroachkvs/testdata/key_schema_key_seeker
index 9509b416..5eff3c64 100644
--- a/cockroachkvs/testdata/key_schema_key_seeker
+++ b/cockroachkvs/testdata/key_schema_key_seeker
@@ -169,3 +169,75 @@ MaterializeUserKey(-1, 0) = hex:6d6f6f0000000000b2d05e0109
 MaterializeUserKey(0, 1) = hex:6d6f6f0000000000b2d05e00000000020d
 MaterializeUserKey(1, 2) = hex:6d6f6f0000000000b2d05e00000000010d
 MaterializeUserKey(2, 3) = hex:6d6f6f0000000000b2d05e0009
+
+define-block
+moo
+moo @ 3000000002,1
+moo @ 3000000001,1
+moo @ 0000000001,0
+----
+Parse(moo) = hex:6d6f6f00
+Parse(moo @ 3000000002,1) = hex:6d6f6f0000000000b2d05e02000000010d
+Parse(moo @ 3000000001,1) = hex:6d6f6f0000000000b2d05e01000000010d
+Parse(moo @ 0000000001,0) = hex:6d6f6f00000000000000000109
+
+# IsLowerBound should return false for all keys because the first key of the
+# block sorts before all the provided keys. Previously, a bug did not account
+# for the empty suffix sorting first.
+
+is-lower-bound
+moo @ 9000000000,2
+moo @ 8000000000,2
+moo @ 8000000000,1
+moo @ 8000000000,0
+moo @ 7000000000,9
+moo @ 3000000000,2
+moo @ 3000000000,1
+moo @ 3000000000,0
+moo @ 0000000000,1
+----
+IsLowerBound(moo @ 9000000000,2, "") = false
+IsLowerBound(moo @ 8000000000,2, "") = false
+IsLowerBound(moo @ 8000000000,1, "") = false
+IsLowerBound(moo @ 8000000000,0, "") = false
+IsLowerBound(moo @ 7000000000,9, "") = false
+IsLowerBound(moo @ 3000000000,2, "") = false
+IsLowerBound(moo @ 3000000000,1, "") = false
+IsLowerBound(moo @ 3000000000,0, "") = false
+IsLowerBound(moo @ 0000000000,1, "") = false
+
+define-block
+moo @ 0000000000,1
+moo @ 3000000002,1
+moo @ 3000000001,1
+moo @ 0000000001,0
+----
+Parse(moo @ 0000000000,1) = hex:6d6f6f000000000000000000000000010d
+Parse(moo @ 3000000002,1) = hex:6d6f6f0000000000b2d05e02000000010d
+Parse(moo @ 3000000001,1) = hex:6d6f6f0000000000b2d05e01000000010d
+Parse(moo @ 0000000001,0) = hex:6d6f6f00000000000000000109
+
+is-lower-bound
+moo
+moo @ 9000000000,2
+moo @ 8000000000,2
+moo @ 8000000000,1
+moo @ 8000000000,0
+moo @ 7000000000,9
+moo @ 3000000000,2
+moo @ 3000000000,1
+moo @ 3000000000,0
+moo @ 0000000000,1
+moo @ 02073a83c45688420eaf97824255790f1e12
+----
+IsLowerBound(moo, "") = true
+IsLowerBound(moo @ 9000000000,2, "") = true
+IsLowerBound(moo @ 8000000000,2, "") = true
+IsLowerBound(moo @ 8000000000,1, "") = true
+IsLowerBound(moo @ 8000000000,0, "") = true
+IsLowerBound(moo @ 7000000000,9, "") = true
+IsLowerBound(moo @ 3000000000,2, "") = true
+IsLowerBound(moo @ 3000000000,1, "") = true
+IsLowerBound(moo @ 3000000000,0, "") = true
+IsLowerBound(moo @ 0000000000,1, "") = true
+IsLowerBound(moo @ 02073a83c45688420eaf97824255790f1e12, "") = true
diff --git a/cockroachkvs/testdata/key_schema_key_writer b/cockroachkvs/testdata/key_schema_key_writer
index 6d7de418..2d62d92d 100644
--- a/cockroachkvs/testdata/key_schema_key_writer
+++ b/cockroachkvs/testdata/key_schema_key_writer
@@ -136,4 +136,4 @@ finish
 +----------+---------------------+---------+---------+
 | hex:fa   | 1730810366077083374 |       0 |         |
 | hex:fa00 | 1730810366080262004 |       0 |         |
-+----------+---------------------+---------+---------+
\ No newline at end of file
++----------+---------------------+---------+---------+
diff --git a/compaction_picker_test.go b/compaction_picker_test.go
index fdd146d0..f1e639c6 100644
--- a/compaction_picker_test.go
+++ b/compaction_picker_test.go
@@ -367,7 +367,7 @@ func TestCompactionPickerEstimatedCompactionDebt(t *testing.T) {
 }
 
 func TestCompactionPickerL0(t *testing.T) {
-	opts := (*Options)(nil).EnsureDefaults()
+	opts := DefaultOptions()
 	opts.Experimental.L0CompactionConcurrency = 1
 
 	parseMeta := func(s string) (*fileMetadata, error) {
@@ -471,12 +471,12 @@ func TestCompactionPickerL0(t *testing.T) {
 						}
 						var compactFile *fileMetadata
 						for _, m := range fileMetas[level] {
-							if m.FileNum == FileNum(fileNum) {
+							if m.FileNum == base.FileNum(fileNum) {
 								compactFile = m
 							}
 						}
 						if compactFile == nil {
-							return fmt.Sprintf("cannot find compaction file %s", FileNum(fileNum))
+							return fmt.Sprintf("cannot find compaction file %s", base.FileNum(fileNum))
 						}
 						compactFile.CompactionState = manifest.CompactionStateCompacting
 						if first || base.InternalCompare(DefaultComparer.Compare, info.largest, compactFile.Largest) < 0 {
@@ -585,7 +585,7 @@ func TestCompactionPickerL0(t *testing.T) {
 }
 
 func TestCompactionPickerConcurrency(t *testing.T) {
-	opts := (*Options)(nil).EnsureDefaults()
+	opts := DefaultOptions()
 	opts.Experimental.L0CompactionConcurrency = 1
 
 	parseMeta := func(s string) (*fileMetadata, error) {
@@ -691,12 +691,12 @@ func TestCompactionPickerConcurrency(t *testing.T) {
 						}
 						var compactFile *fileMetadata
 						for _, m := range fileMetas[level] {
-							if m.FileNum == FileNum(fileNum) {
+							if m.FileNum == base.FileNum(fileNum) {
 								compactFile = m
 							}
 						}
 						if compactFile == nil {
-							return fmt.Sprintf("cannot find compaction file %s", FileNum(fileNum))
+							return fmt.Sprintf("cannot find compaction file %s", base.FileNum(fileNum))
 						}
 						compactFile.CompactionState = manifest.CompactionStateCompacting
 						if first || base.InternalCompare(DefaultComparer.Compare, info.largest, compactFile.Largest) < 0 {
@@ -780,7 +780,7 @@ func TestCompactionPickerConcurrency(t *testing.T) {
 }
 
 func TestCompactionPickerPickReadTriggered(t *testing.T) {
-	opts := (*Options)(nil).EnsureDefaults()
+	opts := DefaultOptions()
 	var picker *compactionPickerByScore
 	var rcList readCompactionQueue
 	var vers *version
@@ -937,8 +937,7 @@ func (d alwaysMultiLevel) allowL0() bool  { return false }
 func (d alwaysMultiLevel) String() string { return "always" }
 
 func TestPickedCompactionSetupInputs(t *testing.T) {
-	opts := &Options{}
-	opts.EnsureDefaults()
+	opts := DefaultOptions()
 
 	parseMeta := func(s string) *fileMetadata {
 		parts := strings.Split(strings.TrimSpace(s), " ")
@@ -1006,7 +1005,7 @@ func TestPickedCompactionSetupInputs(t *testing.T) {
 			pc.startLevel, pc.outputLevel = &pc.inputs[0], &pc.inputs[1]
 			var currentLevel int
 			var files [numLevels][]*fileMetadata
-			fileNum := FileNum(1)
+			fileNum := base.FileNum(1)
 
 			for _, data := range strings.Split(d.Input, "\n") {
 				switch data[:2] {
@@ -1117,8 +1116,7 @@ func TestPickedCompactionSetupInputs(t *testing.T) {
 }
 
 func TestPickedCompactionExpandInputs(t *testing.T) {
-	opts := &Options{}
-	opts.EnsureDefaults()
+	opts := DefaultOptions()
 	cmp := DefaultComparer.Compare
 	var files []*fileMetadata
 
@@ -1146,7 +1144,7 @@ func TestPickedCompactionExpandInputs(t *testing.T) {
 				}
 				for _, data := range strings.Split(d.Input, "\n") {
 					meta := parseMeta(data)
-					meta.FileNum = FileNum(len(files))
+					meta.FileNum = base.FileNum(len(files))
 					files = append(files, meta)
 				}
 				manifest.SortBySmallest(files, cmp)
@@ -1197,7 +1195,7 @@ func TestPickedCompactionExpandInputs(t *testing.T) {
 }
 
 func TestCompactionOutputFileSize(t *testing.T) {
-	opts := (*Options)(nil).EnsureDefaults()
+	opts := DefaultOptions()
 	var picker *compactionPickerByScore
 	var vers *version
 
diff --git a/compaction_test.go b/compaction_test.go
index 6a49f886..516c8724 100644
--- a/compaction_test.go
+++ b/compaction_test.go
@@ -122,8 +122,8 @@ func TestPickCompaction(t *testing.T) {
 		return strings.Join(ss, ",")
 	}
 
-	opts := (*Options)(nil).EnsureDefaults()
-	newFileMeta := func(fileNum FileNum, size uint64, smallest, largest base.InternalKey) *fileMetadata {
+	opts := DefaultOptions()
+	newFileMeta := func(fileNum base.FileNum, size uint64, smallest, largest base.InternalKey) *fileMetadata {
 		m := (&fileMetadata{
 			FileNum: fileNum,
 			Size:    size,
@@ -569,7 +569,8 @@ func (t *cpuPermissionGranter) CPUWorkDone(_ CPUWorkHandle) {
 // the acquired handles are returned.
 func TestCompactionCPUGranter(t *testing.T) {
 	mem := vfs.NewMem()
-	opts := (&Options{FS: mem}).WithFSDefaults()
+	opts := &Options{FS: mem}
+	opts.WithFSDefaults()
 	g := &cpuPermissionGranter{permit: true}
 	opts.Experimental.CPUWorkPermissionGranter = g
 	d, err := Open("", opts)
@@ -590,7 +591,8 @@ func TestCompactionCPUGranter(t *testing.T) {
 // Tests that there's no errors or panics when the default CPU granter is used.
 func TestCompactionCPUGranterDefault(t *testing.T) {
 	mem := vfs.NewMem()
-	opts := (&Options{FS: mem}).WithFSDefaults()
+	opts := &Options{FS: mem}
+	opts.WithFSDefaults()
 	d, err := Open("", opts)
 	if err != nil {
 		t.Fatalf("Open: %v", err)
@@ -619,7 +621,8 @@ func TestCompaction(t *testing.T) {
 		L0CompactionThreshold: 8,
 		MemTableSize:          memTableSize,
 	}
-	opts.testingRandomized(t).WithFSDefaults()
+	opts = opts.testingRandomized(t)
+	opts.WithFSDefaults()
 	d, err := Open("", opts)
 	if err != nil {
 		t.Fatalf("Open: %v", err)
@@ -943,13 +946,14 @@ func TestManualCompaction(t *testing.T) {
 		mem = vfs.NewMem()
 		require.NoError(t, mem.MkdirAll("ext", 0755))
 
-		opts := (&Options{
+		opts := &Options{
 			FS:                          mem,
 			DebugCheck:                  DebugCheckLevels,
 			DisableAutomaticCompactions: true,
 			EventListener:               compactionLogEventListener,
 			FormatMajorVersion:          randVersion(minVersion, maxVersion),
-		}).WithFSDefaults()
+		}
+		opts.WithFSDefaults()
 		opts.Experimental.EnableColumnarBlocks = func() bool { return true }
 
 		var err error
@@ -1047,13 +1051,14 @@ func TestManualCompaction(t *testing.T) {
 				}
 
 				mem = vfs.NewMem()
-				opts := (&Options{
+				opts := &Options{
 					FS:                          mem,
 					DebugCheck:                  DebugCheckLevels,
 					EventListener:               compactionLogEventListener,
 					FormatMajorVersion:          randVersion(minVersion, maxVersion),
 					DisableAutomaticCompactions: true,
-				}).WithFSDefaults()
+				}
+				opts.WithFSDefaults()
 				opts.Experimental.EnableColumnarBlocks = func() bool { return true }
 
 				var err error
@@ -1289,7 +1294,7 @@ func TestManualCompaction(t *testing.T) {
 }
 
 func TestCompactionOutputLevel(t *testing.T) {
-	opts := (*Options)(nil).EnsureDefaults()
+	opts := DefaultOptions()
 	version := manifest.TestingNewVersion(opts.Comparer)
 
 	datadriven.RunTest(t, "testdata/compaction_output_level",
@@ -1341,7 +1346,7 @@ func TestCompactionDeleteOnlyHints(t *testing.T) {
 				return nil, err
 			}
 		}
-		opts := (&Options{
+		opts := &Options{
 			FS:         vfs.NewMem(),
 			DebugCheck: DebugCheckLevels,
 			// Collection of table stats can trigger compactions. As we want full
@@ -1356,7 +1361,8 @@ func TestCompactionDeleteOnlyHints(t *testing.T) {
 				},
 			},
 			FormatMajorVersion: internalFormatNewest,
-		}).WithFSDefaults()
+		}
+		opts.WithFSDefaults()
 		opts.Experimental.EnableDeleteOnlyCompactionExcises = func() bool { return true }
 		opts.Experimental.EnableColumnarBlocks = func() bool { return true }
 		return opts, nil
@@ -1608,7 +1614,7 @@ func TestCompactionTombstones(t *testing.T) {
 						return err.Error()
 					}
 				}
-				opts := (&Options{
+				opts := &Options{
 					FS:         vfs.NewMem(),
 					DebugCheck: DebugCheckLevels,
 					EventListener: &EventListener{
@@ -1617,7 +1623,8 @@ func TestCompactionTombstones(t *testing.T) {
 						},
 					},
 					FormatMajorVersion: internalFormatNewest,
-				}).WithFSDefaults()
+				}
+				opts.WithFSDefaults()
 				opts.Experimental.EnableDeleteOnlyCompactionExcises = func() bool { return true }
 				opts.Experimental.EnableColumnarBlocks = func() bool { return true }
 				var err error
@@ -1812,7 +1819,7 @@ func TestCompactionReadTriggered(t *testing.T) {
 						return err.Error()
 					}
 				}
-				opts := (&Options{
+				opts := &Options{
 					FS:         vfs.NewMem(),
 					DebugCheck: DebugCheckLevels,
 					EventListener: &EventListener{
@@ -1820,7 +1827,8 @@ func TestCompactionReadTriggered(t *testing.T) {
 							compactInfo = &info
 						},
 					},
-				}).WithFSDefaults()
+				}
+				opts.WithFSDefaults()
 				var err error
 				d, err = runDBDefineCmd(td, opts)
 				if err != nil {
@@ -2036,7 +2044,7 @@ func TestCompactionErrorOnUserKeyOverlap(t *testing.T) {
 					formatKey: DefaultComparer.FormatKey,
 				}
 				var files []manifest.NewFileEntry
-				fileNum := FileNum(1)
+				fileNum := base.FileNum(1)
 
 				for _, data := range strings.Split(d.Input, "\n") {
 					meta := parseMeta(data)
@@ -2072,7 +2080,7 @@ func TestCompactionErrorCleanup(t *testing.T) {
 
 	mem := vfs.NewMem()
 	ii := errorfs.OnIndex(math.MaxInt32) // start disabled
-	opts := (&Options{
+	opts := &Options{
 		FS:     errorfs.Wrap(mem, errorfs.ErrInjected.If(ii)),
 		Levels: make([]LevelOptions, numLevels),
 		EventListener: &EventListener{
@@ -2090,7 +2098,8 @@ func TestCompactionErrorCleanup(t *testing.T) {
 				}
 			},
 		},
-	}).WithFSDefaults()
+	}
+	opts.WithFSDefaults()
 	for i := range opts.Levels {
 		opts.Levels[i].TargetFileSize = 1
 	}
@@ -2175,7 +2184,7 @@ func TestCompactionCheckOrdering(t *testing.T) {
 				var sublevel []*fileMetadata
 				var sublevelNum int
 				var parsingSublevel bool
-				fileNum := FileNum(1)
+				fileNum := base.FileNum(1)
 
 				switchSublevel := func() {
 					if sublevel != nil {
@@ -2268,9 +2277,11 @@ func TestCompactFlushQueuedMemTableAndFlushMetrics(t *testing.T) {
 		// Verify that manual compaction forces a flush of a queued memtable.
 
 		mem := vfs.NewMem()
-		d, err := Open("", testingRandomized(t, &Options{
+		opts := &Options{
 			FS: mem,
-		}).WithFSDefaults())
+		}
+		opts.WithFSDefaults()
+		d, err := Open("", testingRandomized(t, opts))
 		require.NoError(t, err)
 
 		// Add the key "a" to the memtable, then fill up the memtable with the key
@@ -2327,9 +2338,11 @@ func TestCompactFlushQueuedLargeBatch(t *testing.T) {
 	// Verify that compaction forces a flush of a queued large batch.
 
 	mem := vfs.NewMem()
-	d, err := Open("", testingRandomized(t, &Options{
+	opts := &Options{
 		FS: mem,
-	}).WithFSDefaults())
+	}
+	opts.WithFSDefaults()
+	d, err := Open("", testingRandomized(t, opts))
 	require.NoError(t, err)
 
 	// The default large batch threshold is slightly less than 1/2 of the
@@ -2367,14 +2380,16 @@ func TestFlushError(t *testing.T) {
 		}
 		return nil
 	}))
-	d, err := Open("", testingRandomized(t, &Options{
+	opts := &Options{
 		FS: fs,
 		EventListener: &EventListener{
 			BackgroundError: func(err error) {
 				t.Log(err)
 			},
 		},
-	}).WithFSDefaults())
+	}
+	opts.WithFSDefaults()
+	d, err := Open("", testingRandomized(t, opts))
 	require.NoError(t, err)
 	require.NoError(t, d.Set([]byte("a"), []byte("foo"), NoSync))
 	require.NoError(t, d.Flush())
@@ -2386,7 +2401,7 @@ func TestAdjustGrandparentOverlapBytesForFlush(t *testing.T) {
 	var lbaseFiles []*manifest.FileMetadata
 	const lbaseSize = 5 << 20
 	for i := 0; i < 100; i++ {
-		m := &manifest.FileMetadata{Size: lbaseSize, FileNum: FileNum(i)}
+		m := &manifest.FileMetadata{Size: lbaseSize, FileNum: base.FileNum(i)}
 		m.InitPhysicalBacking()
 		lbaseFiles =
 			append(lbaseFiles, m)
@@ -2421,9 +2436,11 @@ func TestAdjustGrandparentOverlapBytesForFlush(t *testing.T) {
 }
 
 func TestCompactionInvalidBounds(t *testing.T) {
-	db, err := Open("", testingRandomized(t, &Options{
+	opts := &Options{
 		FS: vfs.NewMem(),
-	}).WithFSDefaults())
+	}
+	opts.WithFSDefaults()
+	db, err := Open("", testingRandomized(t, opts))
 	require.NoError(t, err)
 	defer db.Close()
 	require.NoError(t, db.Compact([]byte("a"), []byte("b"), false))
@@ -2441,7 +2458,7 @@ func TestMarkedForCompaction(t *testing.T) {
 	}()
 
 	var buf bytes.Buffer
-	opts := (&Options{
+	opts := &Options{
 		FS:                          mem,
 		DebugCheck:                  DebugCheckLevels,
 		DisableAutomaticCompactions: true,
@@ -2455,7 +2472,8 @@ func TestMarkedForCompaction(t *testing.T) {
 				fmt.Fprintln(&buf, info)
 			},
 		},
-	}).WithFSDefaults()
+	}
+	opts.WithFSDefaults()
 	opts.Experimental.EnableColumnarBlocks = func() bool { return true }
 
 	reset := func() {
@@ -2629,7 +2647,7 @@ func TestCompaction_LogAndApplyFails(t *testing.T) {
 		var db *DB
 		inj := &createManifestErrorInjector{}
 		logger := &fatalCapturingLogger{t: t}
-		opts := (&Options{
+		opts := &Options{
 			FS: errorfs.Wrap(vfs.NewMem(), inj),
 			// Rotate the manifest after each write. This is required to trigger a
 			// file creation, into which errors can be injected.
@@ -2643,7 +2661,8 @@ func TestCompaction_LogAndApplyFails(t *testing.T) {
 				},
 			},
 			DisableAutomaticCompactions: true,
-		}).WithFSDefaults()
+		}
+		opts.WithFSDefaults()
 
 		db, err := Open("", opts)
 		require.NoError(t, err)
@@ -2767,7 +2786,7 @@ func TestCompactionErrorStats(t *testing.T) {
 
 	mem := vfs.NewMem()
 	injector := &WriteErrorInjector{}
-	opts := (&Options{
+	opts := &Options{
 		FS:     errorfs.Wrap(mem, injector),
 		Levels: make([]LevelOptions, numLevels),
 		EventListener: &EventListener{
@@ -2785,7 +2804,8 @@ func TestCompactionErrorStats(t *testing.T) {
 				}
 			},
 		},
-	}).WithFSDefaults()
+	}
+	opts.WithFSDefaults()
 	for i := range opts.Levels {
 		opts.Levels[i].TargetFileSize = 1
 	}
diff --git a/data_test.go b/data_test.go
index 7f179602..10d803f9 100644
--- a/data_test.go
+++ b/data_test.go
@@ -784,7 +784,10 @@ func runCompactCmd(td *datadriven.TestData, d *DB) error {
 // creation of invalid database states. If opts.DebugCheck is set, the
 // level checker should detect the invalid state.
 func runDBDefineCmd(td *datadriven.TestData, opts *Options) (*DB, error) {
-	opts = opts.EnsureDefaults()
+	if opts == nil {
+		opts = &Options{}
+	}
+	opts.EnsureDefaults()
 	opts.FS = vfs.NewMem()
 	return runDBDefineCmdReuseFS(td, opts)
 }
@@ -792,7 +795,7 @@ func runDBDefineCmd(td *datadriven.TestData, opts *Options) (*DB, error) {
 // runDBDefineCmdReuseFS is like runDBDefineCmd, but does not set opts.FS, expecting
 // the caller to have set an appropriate FS already.
 func runDBDefineCmdReuseFS(td *datadriven.TestData, opts *Options) (*DB, error) {
-	opts = opts.EnsureDefaults()
+	opts.EnsureDefaults()
 	if err := parseDBOptionsArgs(opts, td.CmdArgs); err != nil {
 		return nil, err
 	}
@@ -1164,7 +1167,7 @@ func runSSTablePropertiesCmd(t *testing.T, td *datadriven.TestData, d *DB) strin
 	if m != nil {
 		backingFileNum = m.FileBacking.DiskFileNum
 	}
-	fileName := base.MakeFilename(fileTypeTable, backingFileNum)
+	fileName := base.MakeFilename(base.FileTypeTable, backingFileNum)
 	f, err := d.opts.FS.Open(fileName)
 	if err != nil {
 		return err.Error()
@@ -1178,9 +1181,8 @@ func runSSTablePropertiesCmd(t *testing.T, td *datadriven.TestData, d *DB) strin
 	// and virtual sstables expect this file number to be set. Split out the
 	// opts into fileNum opts, and cache opts.
 	readerOpts.CacheOpts = sstableinternal.CacheOptions{
-		Cache:   d.opts.Cache,
-		CacheID: 0,
-		FileNum: backingFileNum,
+		CacheHandle: d.cacheHandle,
+		FileNum:     backingFileNum,
 	}
 	r, err := sstable.NewReader(context.Background(), readable, readerOpts)
 	if err != nil {
diff --git a/db_test.go b/db_test.go
index ca7154cc..e298572b 100644
--- a/db_test.go
+++ b/db_test.go
@@ -810,11 +810,12 @@ func TestMemTableReservation(t *testing.T) {
 
 	// Add a block to the cache. Note that the memtable size is larger than the
 	// cache size, so opening the DB should cause this block to be evicted.
-	tmpID := opts.Cache.NewID()
 	helloWorld := []byte("hello world")
 	value := cache.Alloc(len(helloWorld))
 	copy(value.RawBuffer(), helloWorld)
-	opts.Cache.Set(tmpID, base.DiskFileNum(0), 0, value)
+	tmpHandle := opts.Cache.NewHandle()
+	defer tmpHandle.Close()
+	tmpHandle.Set(base.DiskFileNum(0), 0, value)
 	value.Release()
 
 	d, err := Open("", opts)
@@ -832,7 +833,7 @@ func TestMemTableReservation(t *testing.T) {
 		t.Fatalf("expected 2 refs, but found %d", refs)
 	}
 	// Verify the memtable reservation has caused our test block to be evicted.
-	if cv := opts.Cache.Get(tmpID, base.DiskFileNum(0), 0); cv != nil {
+	if cv := tmpHandle.Get(base.DiskFileNum(0), 0); cv != nil {
 		t.Fatalf("expected failure, but found success: %#v", cv)
 	}
 
@@ -1036,7 +1037,7 @@ func TestRollManifest(t *testing.T) {
 		if !ok {
 			continue
 		}
-		if fileType == fileTypeManifest {
+		if fileType == base.FileTypeManifest {
 			manifests = append(manifests, filename)
 		}
 	}
diff --git a/external_iterator_test.go b/external_iterator_test.go
index 66fa57ff..130fa905 100644
--- a/external_iterator_test.go
+++ b/external_iterator_test.go
@@ -7,6 +7,7 @@ package pebble
 import (
 	"bytes"
 	"fmt"
+	"math/rand/v2"
 	"testing"
 
 	"github.com/cockroachdb/datadriven"
@@ -71,19 +72,24 @@ func TestExternalIterator(t *testing.T) {
 
 func BenchmarkExternalIter_NonOverlapping_Scan(b *testing.B) {
 	ks := testkeys.Alpha(6)
-	opts := (&Options{Comparer: testkeys.Comparer}).EnsureDefaults()
+	opts := &Options{Comparer: testkeys.Comparer}
+	opts.EnsureDefaults()
 	iterOpts := &IterOptions{
 		KeyTypes: IterKeyTypePointsAndRanges,
 	}
 	writeOpts := opts.MakeWriterOptions(6, sstable.TableFormatPebblev2)
+	var valBuf [512]byte
 
 	for _, keyCount := range []int{100, 10_000, 100_000} {
 		b.Run(fmt.Sprintf("keys=%d", keyCount), func(b *testing.B) {
 			for _, fileCount := range []int{1, 10, 100} {
 				b.Run(fmt.Sprintf("files=%d", fileCount), func(b *testing.B) {
+					prng := rand.New(rand.NewPCG(0, 0))
+
 					var fs vfs.FS = vfs.NewMem()
 					filenames := make([]string, fileCount)
 					var keys [][]byte
+
 					for i := 0; i < fileCount; i++ {
 						filename := fmt.Sprintf("%03d.sst", i)
 						wf, err := fs.Create(filename, vfs.WriteCategoryUnspecified)
@@ -92,7 +98,11 @@ func BenchmarkExternalIter_NonOverlapping_Scan(b *testing.B) {
 						for j := 0; j < keyCount/fileCount; j++ {
 							key := testkeys.Key(ks, int64(len(keys)))
 							keys = append(keys, key)
-							require.NoError(b, w.Set(key, key))
+							for i := range valBuf {
+								valBuf[i] = byte(prng.Uint32())
+							}
+							require.NoError(b, err)
+							require.NoError(b, w.Set(key, valBuf[:]))
 						}
 						require.NoError(b, w.Close())
 						filenames[i] = filename
diff --git a/external_test.go b/external_test.go
index 3707ae15..03fcaebb 100644
--- a/external_test.go
+++ b/external_test.go
@@ -49,7 +49,6 @@ func TestIteratorErrors(t *testing.T) {
 		testOpts.Opts.FS = vfs.NewMem()
 	}
 
-	testOpts.Opts.Cache.Ref()
 	{
 		test, err := metamorphic.New(metamorphic.GenerateOps(
 			rng, 10000, kf, metamorphic.WriteOpConfig()),
diff --git a/file_cache_test.go b/file_cache_test.go
index 0e6672b7..dd15e020 100644
--- a/file_cache_test.go
+++ b/file_cache_test.go
@@ -12,6 +12,7 @@ import (
 	"math/rand/v2"
 	"os"
 	"path"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -20,12 +21,12 @@ import (
 
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/cache"
 	"github.com/cockroachdb/pebble/internal/manifest"
 	"github.com/cockroachdb/pebble/internal/testkeys"
 	"github.com/cockroachdb/pebble/objstorage"
 	"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
 	"github.com/cockroachdb/pebble/sstable"
-	"github.com/cockroachdb/pebble/sstable/block"
 	"github.com/cockroachdb/pebble/vfs"
 	"github.com/stretchr/testify/require"
 )
@@ -74,14 +75,17 @@ func (fs *fileCacheTestFS) Open(name string, opts ...vfs.OpenOption) (vfs.File,
 	return &fileCacheTestFile{f, fs, name}, nil
 }
 
-func (fs *fileCacheTestFS) validate(
-	t *testing.T, c *fileCacheContainer, f func(i, gotO, gotC int) error,
+func (fs *fileCacheTestFS) validateAndCloseHandle(
+	t *testing.T, h *fileCacheHandle, f func(i, gotO, gotC int) error,
 ) {
 	if err := fs.validateOpenTables(f); err != nil {
 		t.Error(err)
 		return
 	}
-	c.close()
+	if err := h.Close(); err != nil {
+		t.Error(err)
+		return
+	}
 	if err := fs.validateNoneStillOpen(); err != nil {
 		t.Error(err)
 		return
@@ -104,7 +108,7 @@ func (fs *fileCacheTestFS) validateOpenTables(f func(i, gotO, gotC int) error) e
 
 		numStillOpen := 0
 		for i := 0; i < fileCacheTestNumTables; i++ {
-			filename := base.MakeFilepath(fs, "", fileTypeTable, base.DiskFileNum(i))
+			filename := base.MakeFilepath(fs, "", base.FileTypeTable, base.DiskFileNum(i))
 			gotO, gotC := fs.openCounts[filename], fs.closeCounts[filename]
 			if gotO > gotC {
 				numStillOpen++
@@ -134,7 +138,7 @@ func (fs *fileCacheTestFS) validateNoneStillOpen() error {
 		defer fs.mu.Unlock()
 
 		for i := 0; i < fileCacheTestNumTables; i++ {
-			filename := base.MakeFilepath(fs, "", fileTypeTable, base.DiskFileNum(i))
+			filename := base.MakeFilepath(fs, "", base.FileTypeTable, base.DiskFileNum(i))
 			gotO, gotC := fs.openCounts[filename], fs.closeCounts[filename]
 			if gotO != gotC {
 				return errors.Errorf("i=%d: opened %d times, closed %d times", i, gotO, gotC)
@@ -149,42 +153,65 @@ const (
 	fileCacheTestCacheSize = 100
 )
 
+type fileCacheTest struct {
+	testing.TB
+	blockCache       *cache.Cache
+	blockCacheHandle *cache.Handle
+	fileCache        *FileCache
+}
+
 // newFileCacheTest returns a shareable file cache to be used for tests.
 // It is the caller's responsibility to unref the file cache.
-func newFileCacheTest(size int64, fileCacheSize int, numShards int) *FileCache {
-	cache := NewCache(size)
-	defer cache.Unref()
-	return NewFileCache(cache, numShards, fileCacheSize)
+func newFileCacheTest(
+	tb testing.TB, blockCacheSize int64, fileCacheSize int, fileCacheNumShards int,
+) *fileCacheTest {
+	blockCache := NewCache(blockCacheSize)
+	blockCacheHandle := blockCache.NewHandle()
+	fileCache := NewFileCache(fileCacheNumShards, fileCacheSize)
+	return &fileCacheTest{
+		TB:               tb,
+		blockCache:       blockCache,
+		blockCacheHandle: blockCacheHandle,
+		fileCache:        fileCache,
+	}
+}
+
+func (t *fileCacheTest) cleanup() {
+	if err := t.fileCache.Unref(); err != nil {
+		t.Error(err)
+	}
+	t.blockCacheHandle.Close()
+	t.blockCache.Unref()
 }
 
-func newFileCacheContainerTest(
-	tc *FileCache, dirname string,
-) (*fileCacheContainer, *fileCacheTestFS, error) {
+// newTestHandle creates a filesystem with a set of test tables and an
+// associated file cache handle. The caller must close the handle.
+func (t *fileCacheTest) newTestHandle() (*fileCacheHandle, *fileCacheTestFS) {
 	xxx := bytes.Repeat([]byte("x"), fileCacheTestNumTables)
 	fs := &fileCacheTestFS{
 		FS: vfs.NewMem(),
 	}
-	objProvider, err := objstorageprovider.Open(objstorageprovider.DefaultSettings(fs, dirname))
+	objProvider, err := objstorageprovider.Open(objstorageprovider.DefaultSettings(fs, ""))
 	if err != nil {
-		return nil, nil, err
+		t.Fatal(err)
 	}
 	defer objProvider.Close()
 
 	for i := 0; i < fileCacheTestNumTables; i++ {
-		w, _, err := objProvider.Create(context.Background(), fileTypeTable, base.DiskFileNum(i), objstorage.CreateOptions{})
+		w, _, err := objProvider.Create(context.Background(), base.FileTypeTable, base.DiskFileNum(i), objstorage.CreateOptions{})
 		if err != nil {
-			return nil, nil, errors.Wrap(err, "fs.Create")
+			t.Fatal(err)
 		}
 		tw := sstable.NewWriter(w, sstable.WriterOptions{TableFormat: sstable.TableFormatPebblev2})
 		ik := base.ParseInternalKey(fmt.Sprintf("k.SET.%d", i))
 		if err := tw.Raw().AddWithForceObsolete(ik, xxx[:i], false); err != nil {
-			return nil, nil, errors.Wrap(err, "tw.Set")
+			t.Fatal(err)
 		}
 		if err := tw.RangeKeySet([]byte("k"), []byte("l"), nil, xxx[:i]); err != nil {
-			return nil, nil, errors.Wrap(err, "tw.Set")
+			t.Fatal(err)
 		}
 		if err := tw.Close(); err != nil {
-			return nil, nil, errors.Wrap(err, "tw.Close")
+			t.Fatal(err)
 		}
 	}
 
@@ -193,64 +220,65 @@ func newFileCacheContainerTest(
 	fs.closeCounts = map[string]int{}
 	fs.mu.Unlock()
 
-	opts := &Options{}
-	opts.EnsureDefaults()
-	if tc == nil {
-		opts.Cache = NewCache(8 << 20) // 8 MB
-		defer opts.Cache.Unref()
-	} else {
-		opts.Cache = tc.cache
+	opts := &Options{
+		Cache:     t.blockCache,
+		FileCache: t.fileCache,
 	}
-
-	c := newFileCacheContainer(tc, opts.Cache.NewID(), objProvider, opts, fileCacheTestCacheSize,
-		&block.CategoryStatsCollector{})
-	return c, fs, nil
+	opts.EnsureDefaults()
+	h := t.fileCache.newHandle(t.blockCacheHandle, objProvider, opts)
+	return h, fs
 }
 
 // Test basic reference counting for the file cache.
 func TestFileCacheRefs(t *testing.T) {
-	c := newFileCacheTest(8<<20, 10, 2)
+	fct := newFileCacheTest(t, 8<<20, 10, 2)
+	// We don't call the full fct.cleanup() method because we will unref the
+	// fileCache in the test.
+	defer fct.blockCacheHandle.Close()
+	defer fct.blockCache.Unref()
+	fc := fct.fileCache
 
-	v := c.refs.Load()
+	v := fc.refs.Load()
 	if v != 1 {
 		require.Equal(t, 1, v)
 	}
 
-	c.Ref()
-	v = c.refs.Load()
+	fc.Ref()
+	v = fc.refs.Load()
 	if v != 2 {
 		require.Equal(t, 2, v)
 	}
 
-	c.Unref()
-	v = c.refs.Load()
+	fc.Unref()
+	v = fc.refs.Load()
 	if v != 1 {
 		require.Equal(t, 1, v)
 	}
 
-	c.Unref()
-	v = c.refs.Load()
+	fc.Unref()
+	v = fc.refs.Load()
 	if v != 0 {
 		require.Equal(t, 0, v)
 	}
 
-	defer func() {
-		if r := recover(); r != nil {
-			if fmt.Sprint(r) != "pebble: inconsistent reference count: -1" {
-				t.Fatalf("unexpected panic message")
+	func() {
+		defer func() {
+			if r := recover(); r != nil {
+				if fmt.Sprint(r) != "pebble: inconsistent reference count: -1" {
+					t.Fatalf("unexpected panic message")
+				}
+			} else if r == nil {
+				t.Fatalf("expected panic")
 			}
-		} else if r == nil {
-			t.Fatalf("expected panic")
-		}
+		}()
+		fc.Unref()
 	}()
-	c.Unref()
+
 }
 
 // Basic test to determine if reads through the file cache are wired correctly.
 func TestVirtualReadsWiring(t *testing.T) {
-	var d *DB
-	var err error
-	d, err = Open("",
+	d, err := Open("",
 		&Options{
 			FS:                 vfs.NewMem(),
 			FormatMajorVersion: internalFormatNewest,
@@ -284,7 +312,7 @@ func TestVirtualReadsWiring(t *testing.T) {
 	l6 := currVersion.Levels[6]
 	l6FileIter := l6.Iter()
 	parentFile := l6FileIter.First()
-	f1 := FileNum(d.mu.versions.nextFileNum.Load())
+	f1 := base.FileNum(d.mu.versions.nextFileNum.Load())
 	f2 := f1 + 1
 	d.mu.versions.nextFileNum.Add(2)
 
@@ -406,11 +434,17 @@ func TestVirtualReadsWiring(t *testing.T) {
 
 // The file cache shouldn't be usable after all the dbs close.
 func TestSharedFileCacheUseAfterAllFree(t *testing.T) {
-	fc := newFileCacheTest(8<<20, 10, 1)
+	fct := newFileCacheTest(t, 8<<20, 10, 1)
+	// We don't call the full fct.cleanup() method because we will unref the
+	// fileCache in the test.
+	defer fct.blockCacheHandle.Close()
+	defer fct.blockCache.Unref()
+	fc := fct.fileCache
+
 	db1, err := Open("test",
 		&Options{
 			FS:        vfs.NewMem(),
-			Cache:     fc.cache,
+			Cache:     fct.blockCache,
 			FileCache: fc,
 		})
 	require.NoError(t, err)
@@ -421,7 +455,7 @@ func TestSharedFileCacheUseAfterAllFree(t *testing.T) {
 	db2, err := Open("test",
 		&Options{
 			FS:        vfs.NewMem(),
-			Cache:     fc.cache,
+			Cache:     fct.blockCache,
 			FileCache: fc,
 		})
 	require.NoError(t, err)
@@ -435,13 +469,9 @@ func TestSharedFileCacheUseAfterAllFree(t *testing.T) {
 	}
 
 	defer func() {
-		// The cache ref gets incremented before the panic, so we should
-		// decrement it to prevent the finalizer from detecting a leak.
-		fc.cache.Unref()
-
 		if r := recover(); r != nil {
 			if fmt.Sprint(r) != "pebble: inconsistent reference count: 1" {
-				t.Fatalf("unexpected panic message")
+				t.Fatalf("unexpected panic message: %v", r)
 			}
 		} else if r == nil {
 			t.Fatalf("expected panic")
@@ -451,7 +481,7 @@ func TestSharedFileCacheUseAfterAllFree(t *testing.T) {
 	db3, _ := Open("test",
 		&Options{
 			FS:        vfs.NewMem(),
-			Cache:     fc.cache,
+			Cache:     fct.blockCache,
 			FileCache: fc,
 		})
 	_ = db3
@@ -460,23 +490,26 @@ func TestSharedFileCacheUseAfterAllFree(t *testing.T) {
 // TestSharedFileCacheUseAfterOneFree tests whether a shared file cache is
 // usable by a db, after one of the db's releases its reference.
 func TestSharedFileCacheUseAfterOneFree(t *testing.T) {
-	tc := newFileCacheTest(8<<20, 10, 1)
+	fct := newFileCacheTest(t, 8<<20, 10, 1)
+	defer fct.cleanup()
+
+	if v := fct.fileCache.refs.Load(); v != 1 {
+		t.Fatalf("expected reference count %d, got %d", 1, v)
+	}
+
 	db1, err := Open("test",
 		&Options{
 			FS:        vfs.NewMem(),
-			Cache:     tc.cache,
-			FileCache: tc,
+			Cache:     fct.blockCache,
+			FileCache: fct.fileCache,
 		})
 	require.NoError(t, err)
 
-	// Release our reference, now that the db has a reference.
-	tc.Unref()
-
 	db2, err := Open("test",
 		&Options{
 			FS:        vfs.NewMem(),
-			Cache:     tc.cache,
-			FileCache: tc,
+			Cache:     fct.blockCache,
+			FileCache: fct.fileCache,
 		})
 	require.NoError(t, err)
 	defer func() {
@@ -486,8 +519,7 @@ func TestSharedFileCacheUseAfterOneFree(t *testing.T) {
 	// Make db1 release a reference to the cache. It should
 	// still be usable by db2.
 	require.NoError(t, db1.Close())
-	v := tc.refs.Load()
-	if v != 1 {
+	if v := fct.fileCache.refs.Load(); v != 2 {
 		t.Fatalf("expected reference count %d, got %d", 1, v)
 	}
 
@@ -503,18 +535,17 @@ func TestSharedFileCacheUseAfterOneFree(t *testing.T) {
 // TestSharedFileCacheUsable ensures that a shared file cache is usable by more
 // than one database at once.
 func TestSharedFileCacheUsable(t *testing.T) {
-	tc := newFileCacheTest(8<<20, 10, 1)
+	fct := newFileCacheTest(t, 8<<20, 10, 1)
+	defer fct.cleanup()
+
 	db1, err := Open("test",
 		&Options{
 			FS:        vfs.NewMem(),
-			Cache:     tc.cache,
-			FileCache: tc,
+			Cache:     fct.blockCache,
+			FileCache: fct.fileCache,
 		})
 	require.NoError(t, err)
 
-	// Release our reference, now that the db has a reference.
-	tc.Unref()
-
 	defer func() {
 		require.NoError(t, db1.Close())
 	}()
@@ -522,8 +553,8 @@ func TestSharedFileCacheUsable(t *testing.T) {
 	db2, err := Open("test",
 		&Options{
 			FS:        vfs.NewMem(),
-			Cache:     tc.cache,
-			FileCache: tc,
+			Cache:     fct.blockCache,
+			FileCache: fct.fileCache,
 		})
 	require.NoError(t, err)
 	defer func() {
@@ -548,18 +579,17 @@ func TestSharedFileCacheUsable(t *testing.T) {
 }
 
 func TestSharedTableConcurrent(t *testing.T) {
-	tc := newFileCacheTest(8<<20, 10, 1)
+	fct := newFileCacheTest(t, 8<<20, 10, 1)
+	defer fct.cleanup()
+
 	db1, err := Open("test",
 		&Options{
 			FS:        vfs.NewMem(),
-			Cache:     tc.cache,
-			FileCache: tc,
+			Cache:     fct.blockCache,
+			FileCache: fct.fileCache,
 		})
 	require.NoError(t, err)
 
-	// Release our reference, now that the db has a reference.
-	tc.Unref()
-
 	defer func() {
 		require.NoError(t, db1.Close())
 	}()
@@ -567,8 +597,8 @@ func TestSharedTableConcurrent(t *testing.T) {
 	db2, err := Open("test",
 		&Options{
 			FS:        vfs.NewMem(),
-			Cache:     tc.cache,
-			FileCache: tc,
+			Cache:     fct.blockCache,
+			FileCache: fct.fileCache,
 		})
 	require.NoError(t, err)
 	defer func() {
@@ -600,8 +630,9 @@ func TestSharedTableConcurrent(t *testing.T) {
 
 func testFileCacheRandomAccess(t *testing.T, concurrent bool) {
 	const N = 2000
-	c, fs, err := newFileCacheContainerTest(nil, "")
-	require.NoError(t, err)
+	fct := newFileCacheTest(t, 8<<20, fileCacheTestCacheSize, runtime.GOMAXPROCS(0))
+	defer fct.cleanup()
+	h, fs := fct.newTestHandle()
 
 	rngMu := sync.Mutex{}
 	rng := rand.New(rand.NewPCG(1, 1))
@@ -612,11 +643,11 @@ func testFileCacheRandomAccess(t *testing.T, concurrent bool) {
 			rngMu.Lock()
 			fileNum, sleepTime := rng.IntN(fileCacheTestNumTables), rng.IntN(1000)
 			rngMu.Unlock()
-			m := &fileMetadata{FileNum: FileNum(fileNum)}
+			m := &fileMetadata{FileNum: base.FileNum(fileNum)}
 			m.InitPhysicalBacking()
 			m.FileBacking.Ref()
 			defer m.FileBacking.Unref()
-			iters, err := c.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
+			iters, err := h.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
 			if err != nil {
 				errc <- errors.Errorf("i=%d, fileNum=%d: find: %v", i, fileNum, err)
 				return
@@ -657,7 +688,7 @@ func testFileCacheRandomAccess(t *testing.T, concurrent bool) {
 			require.NoError(t, <-errc)
 		}
 	}
-	fs.validate(t, c, nil)
+	fs.validateAndCloseHandle(t, h, nil)
 }
 
 func TestFileCacheRandomAccessSequential(t *testing.T) { testFileCacheRandomAccess(t, false) }
@@ -669,20 +700,21 @@ func testFileCacheFrequentlyUsedInternal(t *testing.T, rangeIter bool) {
 		pinned0 = 7
 		pinned1 = 11
 	)
-	c, fs, err := newFileCacheContainerTest(nil, "")
-	require.NoError(t, err)
+	fct := newFileCacheTest(t, 8<<20, fileCacheTestCacheSize, runtime.GOMAXPROCS(0))
+	defer fct.cleanup()
+	h, fs := fct.newTestHandle()
 
 	for i := 0; i < N; i++ {
 		for _, j := range [...]int{pinned0, i % fileCacheTestNumTables, pinned1} {
 			var iters iterSet
 			var err error
-			m := &fileMetadata{FileNum: FileNum(j)}
+			m := &fileMetadata{FileNum: base.FileNum(j)}
 			m.InitPhysicalBacking()
 			m.FileBacking.Ref()
 			if rangeIter {
-				iters, err = c.newIters(context.Background(), m, nil, internalIterOpts{}, iterRangeKeys)
+				iters, err = h.newIters(context.Background(), m, nil, internalIterOpts{}, iterRangeKeys)
 			} else {
-				iters, err = c.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
+				iters, err = h.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
 			}
 			if err != nil {
 				t.Fatalf("i=%d, j=%d: find: %v", i, j, err)
@@ -693,7 +725,7 @@ func testFileCacheFrequentlyUsedInternal(t *testing.T, rangeIter bool) {
 		}
 	}
 
-	fs.validate(t, c, func(i, gotO, gotC int) error {
+	fs.validateAndCloseHandle(t, h, func(i, gotO, gotC int) error {
 		if i == pinned0 || i == pinned1 {
 			if gotO != 1 || gotC != 0 {
 				return errors.Errorf("i=%d: pinned table: got %d, %d, want %d, %d", i, gotO, gotC, 1, 0)
@@ -717,23 +749,22 @@ func TestSharedFileCacheFrequentlyUsed(t *testing.T) {
 		pinned0 = 7
 		pinned1 = 11
 	)
-	tc := newFileCacheTest(8<<20, 2*fileCacheTestCacheSize, 16)
-	c1, fs1, err := newFileCacheContainerTest(tc, "")
-	require.NoError(t, err)
-	c2, fs2, err := newFileCacheContainerTest(tc, "")
-	require.NoError(t, err)
-	tc.Unref()
+	fct := newFileCacheTest(t, 8<<20, 2*fileCacheTestCacheSize, 16)
+	defer fct.cleanup()
+
+	h1, fs1 := fct.newTestHandle()
+	h2, fs2 := fct.newTestHandle()
 
 	for i := 0; i < N; i++ {
 		for _, j := range [...]int{pinned0, i % fileCacheTestNumTables, pinned1} {
-			m := &fileMetadata{FileNum: FileNum(j)}
+			m := &fileMetadata{FileNum: base.FileNum(j)}
 			m.InitPhysicalBacking()
 			m.FileBacking.Ref()
-			iters1, err := c1.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
+			iters1, err := h1.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
 			if err != nil {
 				t.Fatalf("i=%d, j=%d: find: %v", i, j, err)
 			}
-			iters2, err := c2.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
+			iters2, err := h2.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
 			if err != nil {
 				t.Fatalf("i=%d, j=%d: find: %v", i, j, err)
 			}
@@ -747,7 +778,7 @@ func TestSharedFileCacheFrequentlyUsed(t *testing.T) {
 		}
 	}
 
-	fs1.validate(t, c1, func(i, gotO, gotC int) error {
+	fs1.validateAndCloseHandle(t, h1, func(i, gotO, gotC int) error {
 		if i == pinned0 || i == pinned1 {
 			if gotO != 1 || gotC != 0 {
 				return errors.Errorf("i=%d: pinned table: got %d, %d, want %d, %d", i, gotO, gotC, 1, 0)
@@ -756,7 +787,7 @@ func TestSharedFileCacheFrequentlyUsed(t *testing.T) {
 		return nil
 	})
 
-	fs2.validate(t, c2, func(i, gotO, gotC int) error {
+	fs2.validateAndCloseHandle(t, h2, func(i, gotO, gotC int) error {
 		if i == pinned0 || i == pinned1 {
 			if gotO != 1 || gotC != 0 {
 				return errors.Errorf("i=%d: pinned table: got %d, %d, want %d, %d", i, gotO, gotC, 1, 0)
@@ -771,21 +802,22 @@ func testFileCacheEvictionsInternal(t *testing.T, rangeIter bool) {
 		N      = 1000
 		lo, hi = 10, 20
 	)
-	c, fs, err := newFileCacheContainerTest(nil, "")
-	require.NoError(t, err)
+	fct := newFileCacheTest(t, 8<<20, fileCacheTestCacheSize, runtime.GOMAXPROCS(0))
+	defer fct.cleanup()
+	h, fs := fct.newTestHandle()
 
 	rng := rand.New(rand.NewPCG(2, 2))
 	for i := 0; i < N; i++ {
 		j := rng.IntN(fileCacheTestNumTables)
 		var iters iterSet
 		var err error
-		m := &fileMetadata{FileNum: FileNum(j)}
+		m := &fileMetadata{FileNum: base.FileNum(j)}
 		m.InitPhysicalBacking()
 		m.FileBacking.Ref()
 		if rangeIter {
-			iters, err = c.newIters(context.Background(), m, nil, internalIterOpts{}, iterRangeKeys)
+			iters, err = h.newIters(context.Background(), m, nil, internalIterOpts{}, iterRangeKeys)
 		} else {
-			iters, err = c.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
+			iters, err = h.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
 		}
 		if err != nil {
 			t.Fatalf("i=%d, j=%d: find: %v", i, j, err)
@@ -794,12 +826,12 @@ func testFileCacheEvictionsInternal(t *testing.T, rangeIter bool) {
 			t.Fatalf("i=%d, j=%d: close: %v", i, j, err)
 		}
 
-		c.evict(base.DiskFileNum(lo + rng.Uint64N(hi-lo)))
+		h.evict(base.DiskFileNum(lo + rng.Uint64N(hi-lo)))
 	}
 
 	sumEvicted, nEvicted := 0, 0
 	sumSafe, nSafe := 0, 0
-	fs.validate(t, c, func(i, gotO, gotC int) error {
+	fs.validateAndCloseHandle(t, h, func(i, gotO, gotC int) error {
 		if lo <= i && i < hi {
 			sumEvicted += gotO
 			nEvicted++
@@ -834,26 +866,25 @@ func TestSharedFileCacheEvictions(t *testing.T) {
 		N      = 1000
 		lo, hi = 10, 20
 	)
-	tc := newFileCacheTest(8<<20, 2*fileCacheTestCacheSize, 16)
-	c1, fs1, err := newFileCacheContainerTest(tc, "")
-	require.NoError(t, err)
-	c2, fs2, err := newFileCacheContainerTest(tc, "")
-	require.NoError(t, err)
-	tc.Unref()
+	fct := newFileCacheTest(t, 8<<20, 2*fileCacheTestCacheSize, 16)
+	defer fct.cleanup()
+
+	h1, fs1 := fct.newTestHandle()
+	h2, fs2 := fct.newTestHandle()
 
 	// TODO(radu): this test fails on most seeds.
 	rng := rand.New(rand.NewPCG(0, 0))
 	for i := 0; i < N; i++ {
 		j := rng.IntN(fileCacheTestNumTables)
-		m := &fileMetadata{FileNum: FileNum(j)}
+		m := &fileMetadata{FileNum: base.FileNum(j)}
 		m.InitPhysicalBacking()
 		m.FileBacking.Ref()
-		iters1, err := c1.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
+		iters1, err := h1.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
 		if err != nil {
 			t.Fatalf("i=%d, j=%d: find: %v", i, j, err)
 		}
 
-		iters2, err := c2.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
+		iters2, err := h2.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
 		if err != nil {
 			t.Fatalf("i=%d, j=%d: find: %v", i, j, err)
 		}
@@ -866,14 +897,14 @@ func TestSharedFileCacheEvictions(t *testing.T) {
 			t.Fatalf("i=%d, j=%d: close: %v", i, j, err)
 		}
 
-		c1.evict(base.DiskFileNum(lo + rng.Uint64N(hi-lo)))
-		c2.evict(base.DiskFileNum(lo + rng.Uint64N(hi-lo)))
+		h1.evict(base.DiskFileNum(lo + rng.Uint64N(hi-lo)))
+		h2.evict(base.DiskFileNum(lo + rng.Uint64N(hi-lo)))
 	}
 
-	check := func(fs *fileCacheTestFS, c *fileCacheContainer) (float64, float64, float64) {
+	check := func(fs *fileCacheTestFS, h *fileCacheHandle) (float64, float64, float64) {
 		sumEvicted, nEvicted := 0, 0
 		sumSafe, nSafe := 0, 0
-		fs.validate(t, c, func(i, gotO, gotC int) error {
+		fs.validateAndCloseHandle(t, h, func(i, gotO, gotC int) error {
 			if lo <= i && i < hi {
 				sumEvicted += gotO
 				nEvicted++
@@ -893,14 +924,14 @@ func TestSharedFileCacheEvictions(t *testing.T) {
 	// (lo, hi, fileCacheTestCacheSize, fileCacheTestNumTables) = (10, 20, 100, 300),
 	// the ratio seems to converge on roughly 1.5 for large N, compared to 1.0 if we do
 	// not evict any cache entries.
-	if fEvicted, fSafe, ratio := check(fs1, c1); ratio < 1.25 {
+	if fEvicted, fSafe, ratio := check(fs1, h1); ratio < 1.25 {
 		t.Errorf(
 			"evicted tables were opened %.3f times on average, safe tables %.3f, ratio %.3f < 1.250",
 			fEvicted, fSafe, ratio,
 		)
 	}
 
-	if fEvicted, fSafe, ratio := check(fs2, c2); ratio < 1.25 {
+	if fEvicted, fSafe, ratio := check(fs2, h2); ratio < 1.25 {
 		t.Errorf(
 			"evicted tables were opened %.3f times on average, safe tables %.3f, ratio %.3f < 1.250",
 			fEvicted, fSafe, ratio,
@@ -909,17 +940,18 @@ func TestSharedFileCacheEvictions(t *testing.T) {
 }
 
 func TestFileCacheIterLeak(t *testing.T) {
-	c, _, err := newFileCacheContainerTest(nil, "")
-	require.NoError(t, err)
+	fct := newFileCacheTest(t, 8<<20, fileCacheTestCacheSize, runtime.GOMAXPROCS(0))
+	defer fct.cleanup()
+	h, _ := fct.newTestHandle()
 
 	m := &fileMetadata{FileNum: 0}
 	m.InitPhysicalBacking()
 	m.FileBacking.Ref()
 	defer m.FileBacking.Unref()
-	iters, err := c.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
+	iters, err := h.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
 	require.NoError(t, err)
 
-	if err := c.close(); err == nil {
+	if err := h.Close(); err == nil {
 		t.Fatalf("expected failure, but found success")
 	} else if !strings.HasPrefix(err.Error(), "leaked iterators:") {
 		t.Fatalf("expected leaked iterators, but found %+v", err)
@@ -930,23 +962,24 @@ func TestFileCacheIterLeak(t *testing.T) {
 }
 
 func TestSharedFileCacheIterLeak(t *testing.T) {
-	tc := newFileCacheTest(8<<20, 2*fileCacheTestCacheSize, 16)
-	c1, _, err := newFileCacheContainerTest(tc, "")
-	require.NoError(t, err)
-	c2, _, err := newFileCacheContainerTest(tc, "")
-	require.NoError(t, err)
-	c3, _, err := newFileCacheContainerTest(tc, "")
-	require.NoError(t, err)
-	tc.Unref()
+	fct := newFileCacheTest(t, 8<<20, 2*fileCacheTestCacheSize, runtime.GOMAXPROCS(0))
+	// We don't call the full fct.cleanup() method because we will unref the
+	// fileCache in the test.
+	defer fct.blockCacheHandle.Close()
+	defer fct.blockCache.Unref()
+
+	h1, _ := fct.newTestHandle()
+	h2, _ := fct.newTestHandle()
+	h3, _ := fct.newTestHandle()
 
 	m := &fileMetadata{FileNum: 0}
 	m.InitPhysicalBacking()
 	m.FileBacking.Ref()
 	defer m.FileBacking.Unref()
-	iters, err := c1.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
+	iters, err := h1.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
 	require.NoError(t, err)
 
-	if err := c1.close(); err == nil {
+	if err := h1.Close(); err == nil {
 		t.Fatalf("expected failure, but found success")
 	} else if !strings.HasPrefix(err.Error(), "leaked iterators:") {
 		t.Fatalf("expected leaked iterators, but found %+v", err)
@@ -955,12 +988,14 @@ func TestSharedFileCacheIterLeak(t *testing.T) {
 	}
 
 	// Closing c2 shouldn't error out since c2 isn't leaking any iterators.
-	require.NoError(t, c2.close())
+	require.NoError(t, h2.Close())
+
+	fct.fileCache.Unref()
 
 	// Closing c3 should error out since c3 holds the last reference to the
 	// FileCache, and when the FileCache closes, it will detect that there was a
 	// leaked iterator.
-	if err := c3.close(); err == nil {
+	if err := h3.Close(); err == nil {
 		t.Fatalf("expected failure, but found success")
 	} else if !strings.HasPrefix(err.Error(), "leaked iterators:") {
 		t.Fatalf("expected leaked iterators, but found %+v", err)
@@ -973,24 +1008,26 @@ func TestSharedFileCacheIterLeak(t *testing.T) {
 
 func TestFileCacheRetryAfterFailure(t *testing.T) {
 	// Test a retry can succeed after a failure, i.e., errors are not cached.
-	c, fs, err := newFileCacheContainerTest(nil, "")
-	require.NoError(t, err)
+	fct := newFileCacheTest(t, 8<<20, fileCacheTestCacheSize, runtime.GOMAXPROCS(0))
+	defer fct.cleanup()
+	h, fs := fct.newTestHandle()
 
 	fs.setOpenError(true /* enabled */)
 	m := &fileMetadata{FileNum: 0}
 	m.InitPhysicalBacking()
 	m.FileBacking.Ref()
 	defer m.FileBacking.Unref()
-	if _, err = c.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys); err == nil {
+	_, err := h.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
+	if err == nil {
 		t.Fatalf("expected failure, but found success")
 	}
 	require.Equal(t, "pebble: backing file 000000 error: injected error", err.Error())
 	fs.setOpenError(false /* enabled */)
 	var iters iterSet
-	iters, err = c.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
+	iters, err = h.newIters(context.Background(), m, nil, internalIterOpts{}, iterPointKeys)
 	require.NoError(t, err)
 	require.NoError(t, iters.Point().Close())
-	fs.validate(t, c, nil)
+	fs.validateAndCloseHandle(t, h, nil)
 }
 
 func TestFileCacheErrorBadMagicNumber(t *testing.T) {
@@ -1007,17 +1044,20 @@ func TestFileCacheErrorBadMagicNumber(t *testing.T) {
 	const testFileNum = 3
 	objProvider, err := objstorageprovider.Open(objstorageprovider.DefaultSettings(fs, ""))
 	require.NoError(t, err)
-	w, _, err := objProvider.Create(context.Background(), fileTypeTable, testFileNum, objstorage.CreateOptions{})
+	w, _, err := objProvider.Create(context.Background(), base.FileTypeTable, testFileNum, objstorage.CreateOptions{})
 	w.Write(buf)
 	require.NoError(t, w.Finish())
-	opts := &Options{}
+
+	fcs := newFileCacheTest(t, 8<<20 /* 8 MB */, runtime.GOMAXPROCS(0), fileCacheTestCacheSize)
+	defer fcs.cleanup()
+	opts := &Options{
+		Cache:     fcs.blockCache,
+		FileCache: fcs.fileCache,
+	}
 	opts.EnsureDefaults()
-	opts.Cache = NewCache(8 << 20) // 8 MB
-	defer opts.Cache.Unref()
-	c := newFileCacheContainer(nil, opts.Cache.NewID(), objProvider, opts, fileCacheTestCacheSize,
-		&block.CategoryStatsCollector{})
+	c := opts.FileCache.newHandle(fcs.blockCacheHandle, objProvider, opts)
 	require.NoError(t, err)
-	defer c.close()
+	defer c.Close()
 
 	m := &fileMetadata{FileNum: testFileNum}
 	m.InitPhysicalBacking()
@@ -1071,27 +1111,26 @@ func TestFileCacheClockPro(t *testing.T) {
 
 	makeTable := func(dfn base.DiskFileNum) {
 		require.NoError(t, err)
-		f, _, err := objProvider.Create(context.Background(), fileTypeTable, dfn, objstorage.CreateOptions{})
+		f, _, err := objProvider.Create(context.Background(), base.FileTypeTable, dfn, objstorage.CreateOptions{})
 		require.NoError(t, err)
 		w := sstable.NewWriter(f, sstable.WriterOptions{})
 		require.NoError(t, w.Set([]byte("a"), nil))
 		require.NoError(t, w.Close())
 	}
 
+	// NB: The file cache size of 200 with a single shard is required for the
+	// expected test values.
+	fcs := newFileCacheTest(t, 8<<20 /* 8 MB */, 200, 1)
+	defer fcs.cleanup()
+
 	opts := &Options{
-		Cache: NewCache(8 << 20), // 8 MB
+		Cache:           fcs.blockCache,
+		FileCache:       fcs.fileCache,
+		LoggerAndTracer: &base.LoggerWithNoopTracer{Logger: base.DefaultLogger},
 	}
 	opts.EnsureDefaults()
-	defer opts.Cache.Unref()
-
-	cache := &fileCacheShard{}
-	// NB: The file cache size of 200 is required for the expected test values.
-	cache.init(200)
-	dbOpts := &fileCacheOpts{}
-	dbOpts.loggerAndTracer = &base.LoggerWithNoopTracer{Logger: opts.Logger}
-	dbOpts.cacheID = 0
-	dbOpts.objProvider = objProvider
-	dbOpts.readerOpts = opts.MakeReaderOptions()
+	h := fcs.fileCache.newHandle(fcs.blockCacheHandle, objProvider, opts)
+	defer h.Close()
 
 	scanner := bufio.NewScanner(f)
 	tables := make(map[int]bool)
@@ -1110,14 +1149,15 @@ func TestFileCacheClockPro(t *testing.T) {
 			tables[key] = true
 		}
 
-		oldHits := cache.hits.Load()
-		m := &fileMetadata{FileNum: FileNum(key)}
+		shard := fcs.fileCache.shards[0]
+		oldHits := shard.hits.Load()
+		m := &fileMetadata{FileNum: base.FileNum(key)}
 		m.InitPhysicalBacking()
 		m.FileBacking.Ref()
-		v := cache.findNode(context.Background(), m.FileBacking, dbOpts)
-		cache.unrefValue(v)
+		v := shard.findNode(context.Background(), m.FileBacking, h)
+		shard.unrefValue(v)
 
-		hit := cache.hits.Load() != oldHits
+		hit := shard.hits.Load() != oldHits
 		wantHit := fields[1][0] == 'h'
 		if hit != wantHit {
 			t.Errorf("%d: cache hit mismatch: got %v, want %v\n", line, hit, wantHit)
@@ -1209,26 +1249,26 @@ func BenchmarkFileCacheHotPath(b *testing.B) {
 
 	makeTable := func(dfn base.DiskFileNum) {
 		require.NoError(b, err)
-		f, _, err := objProvider.Create(context.Background(), fileTypeTable, dfn, objstorage.CreateOptions{})
+		f, _, err := objProvider.Create(context.Background(), base.FileTypeTable, dfn, objstorage.CreateOptions{})
 		require.NoError(b, err)
 		w := sstable.NewWriter(f, sstable.WriterOptions{})
 		require.NoError(b, w.Set([]byte("a"), nil))
 		require.NoError(b, w.Close())
 	}
 
+	fcs := newFileCacheTest(b, 8<<20 /* 8 MB */, 2, 1)
+	defer fcs.cleanup()
+
 	opts := &Options{
-		Cache: NewCache(8 << 20), // 8 MB
+		Cache:           fcs.blockCache,
+		FileCache:       fcs.fileCache,
+		LoggerAndTracer: &base.LoggerWithNoopTracer{Logger: base.DefaultLogger},
 	}
 	opts.EnsureDefaults()
-	defer opts.Cache.Unref()
+	h := fcs.fileCache.newHandle(fcs.blockCacheHandle, objProvider, opts)
+	defer h.Close()
 
-	cache := &fileCacheShard{}
-	cache.init(2)
-	dbOpts := &fileCacheOpts{}
-	dbOpts.loggerAndTracer = &base.LoggerWithNoopTracer{Logger: opts.Logger}
-	dbOpts.cacheID = 0
-	dbOpts.objProvider = objProvider
-	dbOpts.readerOpts = opts.MakeReaderOptions()
+	shard := fcs.fileCache.shards[0]
 
 	makeTable(1)
 
@@ -1238,8 +1278,8 @@ func BenchmarkFileCacheHotPath(b *testing.B) {
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		v := cache.findNode(context.Background(), m.FileBacking, dbOpts)
-		cache.unrefValue(v)
+		v := shard.findNode(context.Background(), m.FileBacking, h)
+		shard.unrefValue(v)
 	}
 }
 
diff --git a/filenames_test.go b/filenames_test.go
index b8dcba0c..683333c6 100644
--- a/filenames_test.go
+++ b/filenames_test.go
@@ -75,7 +75,7 @@ func allTempFiles(t *testing.T, fs vfs.FS) []string {
 	require.NoError(t, err)
 	for _, f := range ls {
 		ft, _, ok := base.ParseFilename(fs, f)
-		if ok && ft == fileTypeTemp {
+		if ok && ft == base.FileTypeTemp {
 			files = append(files, f)
 		}
 	}
diff --git a/flushable_test.go b/flushable_test.go
index 68493806..50aab42d 100644
--- a/flushable_test.go
+++ b/flushable_test.go
@@ -57,7 +57,7 @@ func TestIngestedSSTFlushableAPI(t *testing.T) {
 
 		// We can reuse the ingestLoad function for this test even if we're
 		// not actually ingesting a file.
-		lr, err := ingestLoad(context.Background(), d.opts, d.FormatMajorVersion(), paths, nil, nil, d.cacheID, pendingOutputs)
+		lr, err := ingestLoad(context.Background(), d.opts, d.FormatMajorVersion(), paths, nil, nil, d.cacheHandle, pendingOutputs)
 		if err != nil {
 			t.Fatal(err)
 		}
diff --git a/format_major_version_test.go b/format_major_version_test.go
index ba64dc55..78c10244 100644
--- a/format_major_version_test.go
+++ b/format_major_version_test.go
@@ -42,7 +42,9 @@ func TestFormatMajorVersion_MigrationDefined(t *testing.T) {
 
 func TestRatchetFormat(t *testing.T) {
 	fs := vfs.NewMem()
-	d, err := Open("", (&Options{FS: fs}).WithFSDefaults())
+	opts := &Options{FS: fs}
+	opts.WithFSDefaults()
+	d, err := Open("", opts)
 	require.NoError(t, err)
 	require.NoError(t, d.Set([]byte("foo"), []byte("bar"), Sync))
 	require.Equal(t, FormatFlushableIngest, d.FormatMajorVersion())
@@ -63,7 +65,9 @@ func TestRatchetFormat(t *testing.T) {
 
 	// If we Open the database again, leaving the default format, the
 	// database should Open using the persisted FormatNewest.
-	d, err = Open("", (&Options{FS: fs, Logger: testLogger{t}}).WithFSDefaults())
+	opts = &Options{FS: fs, Logger: testLogger{t}}
+	opts.WithFSDefaults()
+	d, err = Open("", opts)
 	require.NoError(t, err)
 	require.Equal(t, internalFormatNewest, d.FormatMajorVersion())
 	require.NoError(t, d.Close())
@@ -74,10 +78,12 @@ func TestRatchetFormat(t *testing.T) {
 	require.NoError(t, m.Move("999999"))
 	require.NoError(t, m.Close())
 
-	_, err = Open("", (&Options{
+	opts = &Options{
 		FS:                 fs,
 		FormatMajorVersion: FormatMinSupported,
-	}).WithFSDefaults())
+	}
+	opts.WithFSDefaults()
+	_, err = Open("", opts)
 	require.Error(t, err)
 	require.EqualError(t, err, `pebble: database "" written in unknown format major version 999999`)
 }
@@ -108,11 +114,12 @@ func TestFormatMajorVersions(t *testing.T) {
 	for vers := FormatMinSupported; vers <= FormatNewest; vers++ {
 		t.Run(fmt.Sprintf("vers=%03d", vers), func(t *testing.T) {
 			fs := vfs.NewCrashableMem()
-			opts := (&Options{
+			opts := &Options{
 				FS:                 fs,
 				FormatMajorVersion: vers,
 				Logger:             testLogger{t},
-			}).WithFSDefaults()
+			}
+			opts.WithFSDefaults()
 
 			// Create a database at this format major version and perform
 			// some very basic operations.
diff --git a/get_iter_test.go b/get_iter_test.go
index 271190ca..507d2c0b 100644
--- a/get_iter_test.go
+++ b/get_iter_test.go
@@ -20,7 +20,7 @@ func TestGetIter(t *testing.T) {
 	// Each element of data is a string of the form "internalKey value".
 	type testTable struct {
 		level   int
-		fileNum FileNum
+		fileNum base.FileNum
 		data    []string
 	}
 
@@ -384,7 +384,7 @@ func TestGetIter(t *testing.T) {
 		desc := tc.description[:strings.Index(tc.description, ":")]
 
 		// m is a map from file numbers to DBs.
-		m := map[FileNum]*memTable{}
+		m := map[base.FileNum]*memTable{}
 		newIter := func(
 			_ context.Context, file *manifest.FileMetadata, _ *IterOptions, _ internalIterOpts, _ iterKinds,
 		) (iterSet, error) {
diff --git a/ingest_test.go b/ingest_test.go
index 7c1a777a..93ab1bce 100644
--- a/ingest_test.go
+++ b/ingest_test.go
@@ -125,11 +125,12 @@ func TestIngestLoad(t *testing.T) {
 				return err.Error()
 			}
 
-			opts := (&Options{
+			opts := &Options{
 				Comparer: DefaultComparer,
 				FS:       mem,
-			}).WithFSDefaults()
-			lr, err := ingestLoad(context.Background(), opts, dbVersion, []string{"ext"}, nil, nil, 0, []base.FileNum{1})
+			}
+			opts.WithFSDefaults()
+			lr, err := ingestLoad(context.Background(), opts, dbVersion, []string{"ext"}, nil, nil, nil, []base.FileNum{1})
 			if err != nil {
 				return err.Error()
 			}
@@ -216,11 +217,13 @@ func TestIngestLoadRand(t *testing.T) {
 		}()
 	}
 
-	opts := (&Options{
+	opts := &Options{
 		Comparer: base.DefaultComparer,
 		FS:       mem,
-	}).WithFSDefaults().EnsureDefaults()
-	lr, err := ingestLoad(context.Background(), opts, version, paths, nil, nil, 0, pending)
+	}
+	opts.WithFSDefaults()
+	opts.EnsureDefaults()
+	lr, err := ingestLoad(context.Background(), opts, version, paths, nil, nil, nil, pending)
 	require.NoError(t, err)
 
 	for _, m := range lr.local {
@@ -236,11 +239,12 @@ func TestIngestLoadInvalid(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, f.Close())
 
-	opts := (&Options{
+	opts := &Options{
 		Comparer: DefaultComparer,
 		FS:       mem,
-	}).WithFSDefaults()
-	if _, err := ingestLoad(context.Background(), opts, internalFormatNewest, []string{"invalid"}, nil, nil, 0, []base.FileNum{1}); err == nil {
+	}
+	opts.WithFSDefaults()
+	if _, err := ingestLoad(context.Background(), opts, internalFormatNewest, []string{"invalid"}, nil, nil, nil, []base.FileNum{1}); err == nil {
 		t.Fatalf("expected error, but found success")
 	}
 }
@@ -308,7 +312,8 @@ func TestIngestLink(t *testing.T) {
 	for i := 0; i <= count; i++ {
 		t.Run("", func(t *testing.T) {
 			opts := &Options{FS: vfs.NewMem()}
-			opts.EnsureDefaults().WithFSDefaults()
+			opts.EnsureDefaults()
+			opts.WithFSDefaults()
 			require.NoError(t, opts.FS.MkdirAll(dir, 0755))
 			objProvider, err := objstorageprovider.Open(objstorageprovider.DefaultSettings(opts.FS, dir))
 			require.NoError(t, err)
@@ -319,7 +324,7 @@ func TestIngestLink(t *testing.T) {
 			for j := range meta {
 				meta[j].path = fmt.Sprintf("external%d", j)
 				meta[j].fileMetadata = &fileMetadata{}
-				meta[j].FileNum = FileNum(j)
+				meta[j].FileNum = base.FileNum(j)
 				meta[j].InitPhysicalBacking()
 				f, err := opts.FS.Create(meta[j].path, vfs.WriteCategoryUnspecified)
 				require.NoError(t, err)
@@ -364,7 +369,7 @@ func TestIngestLink(t *testing.T) {
 					if !ok {
 						t.Fatalf("unable to parse filename: %s", files[j])
 					}
-					if fileTypeTable != ftype {
+					if base.FileTypeTable != ftype {
 						t.Fatalf("expected table, but found %d", ftype)
 					}
 					if j != int(fileNum) {
@@ -393,7 +398,8 @@ func TestIngestLinkFallback(t *testing.T) {
 	require.NoError(t, err)
 
 	opts := &Options{FS: errorfs.Wrap(mem, errorfs.ErrInjected.If(errorfs.OnIndex(1)))}
-	opts.EnsureDefaults().WithFSDefaults()
+	opts.EnsureDefaults()
+	opts.WithFSDefaults()
 	objSettings := objstorageprovider.DefaultSettings(opts.FS, "")
 	// Prevent the provider from listing the dir (where we may get an injected error).
 	objSettings.FSDirInitialListing = []string{}
@@ -454,7 +460,7 @@ func TestOverlappingIngestedSSTs(t *testing.T) {
 		}
 
 		require.NoError(t, mem.MkdirAll("ext", 0755))
-		opts = (&Options{
+		opts = &Options{
 			FS: vfs.WithLogging(mem, func(format string, args ...interface{}) {
 				fsLog.Lock()
 				defer fsLog.Unlock()
@@ -467,7 +473,8 @@ func TestOverlappingIngestedSSTs(t *testing.T) {
 			DebugCheck:                  DebugCheckLevels,
 			FormatMajorVersion:          internalFormatNewest,
 			Logger:                      testLogger{t},
-		}).WithFSDefaults()
+		}
+		opts.WithFSDefaults()
 		opts.Experimental.EnableColumnarBlocks = func() bool { return true }
 		if testing.Verbose() {
 			lel := MakeLoggingEventListener(DefaultLogger)
@@ -1371,7 +1378,8 @@ func TestSimpleIngestShared(t *testing.T) {
 	{
 		// Create a shared file.
 		fn := base.FileNum(2)
-		f, meta, err := provider2.Create(context.TODO(), fileTypeTable, base.PhysicalTableDiskFileNum(fn), objstorage.CreateOptions{PreferSharedStorage: true})
+		f, meta, err := provider2.Create(context.TODO(), base.FileTypeTable, base.PhysicalTableDiskFileNum(fn),
+			objstorage.CreateOptions{PreferSharedStorage: true})
 		require.NoError(t, err)
 		w := sstable.NewWriter(f, d.opts.MakeWriterOptions(0, d.TableFormat()))
 		w.Set([]byte("d"), []byte("shared"))
@@ -2174,7 +2182,8 @@ func TestIngestMemtableOverlaps(t *testing.T) {
 					opts := &Options{
 						Comparer: &comparer,
 					}
-					opts.EnsureDefaults().WithFSDefaults()
+					opts.EnsureDefaults()
+					opts.WithFSDefaults()
 					if len(d.CmdArgs) > 1 {
 						return fmt.Sprintf("%s expects at most 1 argument", d.Cmd)
 					}
diff --git a/internal/arenaskl/race_test.go b/internal/arenaskl/race_test.go
index b9310c92..dfff4957 100644
--- a/internal/arenaskl/race_test.go
+++ b/internal/arenaskl/race_test.go
@@ -1,5 +1,4 @@
 //go:build race
-// +build race
 
 // Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
 // of this source code is governed by a BSD-style license that can be found in
diff --git a/internal/arenaskl/skl_test.go b/internal/arenaskl/skl_test.go
index db30bc84..ff0965f8 100644
--- a/internal/arenaskl/skl_test.go
+++ b/internal/arenaskl/skl_test.go
@@ -34,69 +34,6 @@ import (
 
 const arenaSize = 1 << 20
 
-// iterAdapter adapts the new Iterator API which returns the key and value from
-// positioning methods (Seek*, First, Last, Next, Prev) to the old API which
-// returned a boolean corresponding to Valid. Only used by test code.
-type iterAdapter struct {
-	*Iterator
-	kv *base.InternalKV
-}
-
-func newIterAdapter(iter *Iterator) *iterAdapter {
-	return &iterAdapter{
-		Iterator: iter,
-	}
-}
-
-func (i *iterAdapter) update(kv *base.InternalKV) bool {
-	i.kv = kv
-	return i.kv != nil
-}
-
-func (i *iterAdapter) String() string {
-	return "iter-adapter"
-}
-
-func (i *iterAdapter) SeekGE(key []byte, flags base.SeekGEFlags) bool {
-	return i.update(i.Iterator.SeekGE(key, flags))
-}
-
-func (i *iterAdapter) SeekPrefixGE(prefix, key []byte, flags base.SeekGEFlags) bool {
-	return i.update(i.Iterator.SeekPrefixGE(prefix, key, flags))
-}
-
-func (i *iterAdapter) SeekLT(key []byte, flags base.SeekLTFlags) bool {
-	return i.update(i.Iterator.SeekLT(key, flags))
-}
-
-func (i *iterAdapter) First() bool {
-	return i.update(i.Iterator.First())
-}
-
-func (i *iterAdapter) Last() bool {
-	return i.update(i.Iterator.Last())
-}
-
-func (i *iterAdapter) Next() bool {
-	return i.update(i.Iterator.Next())
-}
-
-func (i *iterAdapter) Prev() bool {
-	return i.update(i.Iterator.Prev())
-}
-
-func (i *iterAdapter) Key() base.InternalKey {
-	return i.kv.K
-}
-
-func (i *iterAdapter) Value() []byte {
-	return i.kv.V.InPlaceValue()
-}
-
-func (i *iterAdapter) Valid() bool {
-	return i.kv != nil
-}
-
 func makeIntKey(i int) base.InternalKey {
 	return base.InternalKey{UserKey: []byte(fmt.Sprintf("%05d", i))}
 }
@@ -123,42 +60,30 @@ func makeInserterAdd(s *Skiplist) func(key base.InternalKey, value []byte) error
 // length iterates over skiplist to give exact size.
 func length(s *Skiplist) int {
 	count := 0
-
-	it := newIterAdapter(s.NewIter(nil, nil))
-	for valid := it.First(); valid; valid = it.Next() {
+	it := s.NewIter(nil, nil)
+	for kv := it.First(); kv != nil; kv = it.Next() {
 		count++
 	}
-
 	return count
 }
 
 // length iterates over skiplist in reverse order to give exact size.
 func lengthRev(s *Skiplist) int {
 	count := 0
-
-	it := newIterAdapter(s.NewIter(nil, nil))
-	for valid := it.Last(); valid; valid = it.Prev() {
+	it := s.NewIter(nil, nil)
+	for kv := it.Last(); kv != nil; kv = it.Prev() {
 		count++
 	}
-
 	return count
 }
 
 func TestEmpty(t *testing.T) {
 	key := makeKey("aaa")
 	l := NewSkiplist(newArena(arenaSize), bytes.Compare)
-	it := newIterAdapter(l.NewIter(nil, nil))
-
-	require.False(t, it.Valid())
-
-	it.First()
-	require.False(t, it.Valid())
-
-	it.Last()
-	require.False(t, it.Valid())
-
-	require.False(t, it.SeekGE(key, base.SeekGEFlagsNone))
-	require.False(t, it.Valid())
+	it := l.NewIter(nil, nil)
+	require.Nil(t, it.First())
+	require.Nil(t, it.Last())
+	require.Nil(t, it.SeekGE(key, base.SeekGEFlagsNone))
 }
 
 func TestFull(t *testing.T) {
@@ -179,12 +104,18 @@ func TestFull(t *testing.T) {
 	require.Equal(t, ErrArenaFull, err)
 }
 
+func mustGetValue(t *testing.T, lv base.LazyValue) []byte {
+	v, _, err := lv.Value(nil)
+	require.NoError(t, err)
+	return v
+}
+
 // TestBasic tests single-threaded seeks and adds.
 func TestBasic(t *testing.T) {
 	for _, inserter := range []bool{false, true} {
 		t.Run(fmt.Sprintf("inserter=%t", inserter), func(t *testing.T) {
 			l := NewSkiplist(newArena(arenaSize), bytes.Compare)
-			it := newIterAdapter(l.NewIter(nil, nil))
+			it := l.NewIter(nil, nil)
 
 			add := l.Add
 			if inserter {
@@ -196,21 +127,24 @@ func TestBasic(t *testing.T) {
 			add(makeIkey("key3"), makeValue(3))
 			add(makeIkey("key2"), makeValue(2))
 
-			require.True(t, it.SeekGE(makeKey("key"), base.SeekGEFlagsNone))
-			require.True(t, it.Valid())
-			require.NotEqual(t, "key", it.Key().UserKey)
+			kv := it.SeekGE(makeKey("key"), base.SeekGEFlagsNone)
+			require.NotNil(t, kv)
+			require.NotEqual(t, "key", kv.K.UserKey)
 
-			require.True(t, it.SeekGE(makeKey("key1"), base.SeekGEFlagsNone))
-			require.EqualValues(t, "key1", it.Key().UserKey)
-			require.EqualValues(t, makeValue(1), it.Value())
+			kv = it.SeekGE(makeKey("key1"), base.SeekGEFlagsNone)
+			require.NotNil(t, kv)
+			require.EqualValues(t, "key1", kv.K.UserKey)
+			require.EqualValues(t, makeValue(1), mustGetValue(t, kv.V))
 
-			require.True(t, it.SeekGE(makeKey("key2"), base.SeekGEFlagsNone))
-			require.EqualValues(t, "key2", it.Key().UserKey)
-			require.EqualValues(t, makeValue(2), it.Value())
+			kv = it.SeekGE(makeKey("key2"), base.SeekGEFlagsNone)
+			require.NotNil(t, kv)
+			require.EqualValues(t, "key2", kv.K.UserKey)
+			require.EqualValues(t, makeValue(2), mustGetValue(t, kv.V))
 
-			require.True(t, it.SeekGE(makeKey("key3"), base.SeekGEFlagsNone))
-			require.EqualValues(t, "key3", it.Key().UserKey)
-			require.EqualValues(t, makeValue(3), it.Value())
+			kv = it.SeekGE(makeKey("key3"), base.SeekGEFlagsNone)
+			require.NotNil(t, kv)
+			require.EqualValues(t, "key3", kv.K.UserKey)
+			require.EqualValues(t, makeValue(3), mustGetValue(t, kv.V))
 
 			key := makeIkey("a")
 			key.SetSeqNum(1)
@@ -218,15 +152,15 @@ func TestBasic(t *testing.T) {
 			key.SetSeqNum(2)
 			add(key, nil)
 
-			require.True(t, it.SeekGE(makeKey("a"), base.SeekGEFlagsNone))
-			require.True(t, it.Valid())
-			require.EqualValues(t, "a", it.Key().UserKey)
-			require.EqualValues(t, 2, it.Key().SeqNum())
+			kv = it.SeekGE(makeKey("a"), base.SeekGEFlagsNone)
+			require.NotNil(t, kv)
+			require.EqualValues(t, "a", kv.K.UserKey)
+			require.EqualValues(t, 2, kv.K.SeqNum())
 
-			require.True(t, it.Next())
-			require.True(t, it.Valid())
-			require.EqualValues(t, "a", it.Key().UserKey)
-			require.EqualValues(t, 1, it.Key().SeqNum())
+			kv = it.Next()
+			require.NotNil(t, kv)
+			require.EqualValues(t, "a", kv.K.UserKey)
+			require.EqualValues(t, 1, kv.K.SeqNum())
 
 			key = makeIkey("b")
 			key.SetSeqNum(2)
@@ -234,15 +168,15 @@ func TestBasic(t *testing.T) {
 			key.SetSeqNum(1)
 			add(key, nil)
 
-			require.True(t, it.SeekGE(makeKey("b"), base.SeekGEFlagsNone))
-			require.True(t, it.Valid())
-			require.EqualValues(t, "b", it.Key().UserKey)
-			require.EqualValues(t, 2, it.Key().SeqNum())
+			kv = it.SeekGE(makeKey("b"), base.SeekGEFlagsNone)
+			require.NotNil(t, kv)
+			require.EqualValues(t, "b", kv.K.UserKey)
+			require.EqualValues(t, 2, kv.K.SeqNum())
 
-			require.True(t, it.Next())
-			require.True(t, it.Valid())
-			require.EqualValues(t, "b", it.Key().UserKey)
-			require.EqualValues(t, 1, it.Key().SeqNum())
+			kv = it.Next()
+			require.NotNil(t, kv)
+			require.EqualValues(t, "b", kv.K.UserKey)
+			require.EqualValues(t, 1, kv.K.SeqNum())
 		})
 	}
 }
@@ -279,9 +213,10 @@ func TestConcurrentBasic(t *testing.T) {
 				go func(i int) {
 					defer wg.Done()
 
-					it := newIterAdapter(l.NewIter(nil, nil))
-					require.True(t, it.SeekGE(makeKey(fmt.Sprintf("%05d", i)), base.SeekGEFlagsNone))
-					require.EqualValues(t, fmt.Sprintf("%05d", i), it.Key().UserKey)
+					it := l.NewIter(nil, nil)
+					kv := it.SeekGE(makeKey(fmt.Sprintf("%05d", i)), base.SeekGEFlagsNone)
+					require.NotNil(t, kv)
+					require.EqualValues(t, fmt.Sprintf("%05d", i), kv.K.UserKey)
 				}(i)
 			}
 			wg.Wait()
@@ -332,13 +267,13 @@ func TestConcurrentOneKey(t *testing.T) {
 				go func() {
 					defer wg.Done()
 
-					it := newIterAdapter(l.NewIter(nil, nil))
-					it.SeekGE(key, base.SeekGEFlagsNone)
-					require.True(t, it.Valid())
-					require.True(t, bytes.Equal(key, it.Key().UserKey))
+					it := l.NewIter(nil, nil)
+					kv := it.SeekGE(key, base.SeekGEFlagsNone)
+					require.NotNil(t, kv)
+					require.True(t, bytes.Equal(key, kv.K.UserKey))
 
 					sawValue.Add(1)
-					v, err := strconv.Atoi(string(it.Value()[1:]))
+					v, err := strconv.Atoi(string(mustGetValue(t, kv.V)[1:]))
 					require.NoError(t, err)
 					require.True(t, 0 <= v && v < n)
 				}()
@@ -355,7 +290,7 @@ func TestSkiplistAdd(t *testing.T) {
 	for _, inserter := range []bool{false, true} {
 		t.Run(fmt.Sprintf("inserter=%t", inserter), func(t *testing.T) {
 			l := NewSkiplist(newArena(arenaSize), bytes.Compare)
-			it := newIterAdapter(l.NewIter(nil, nil))
+			it := l.NewIter(nil, nil)
 
 			add := l.Add
 			if inserter {
@@ -365,12 +300,13 @@ func TestSkiplistAdd(t *testing.T) {
 			// Add nil key and value (treated same as empty).
 			err := add(base.InternalKey{}, nil)
 			require.Nil(t, err)
-			require.True(t, it.SeekGE([]byte{}, base.SeekGEFlagsNone))
-			require.EqualValues(t, []byte{}, it.Key().UserKey)
-			require.EqualValues(t, []byte{}, it.Value())
+			kv := it.SeekGE([]byte{}, base.SeekGEFlagsNone)
+			require.NotNil(t, kv)
+			require.EqualValues(t, []byte{}, kv.K.UserKey)
+			require.EqualValues(t, []byte{}, mustGetValue(t, kv.V))
 
 			l = NewSkiplist(newArena(arenaSize), bytes.Compare)
-			it = newIterAdapter(l.NewIter(nil, nil))
+			it = l.NewIter(nil, nil)
 
 			add = l.Add
 			if inserter {
@@ -380,43 +316,46 @@ func TestSkiplistAdd(t *testing.T) {
 			// Add empty key and value (treated same as nil).
 			err = add(makeIkey(""), []byte{})
 			require.Nil(t, err)
-			require.True(t, it.SeekGE([]byte{}, base.SeekGEFlagsNone))
-			require.EqualValues(t, []byte{}, it.Key().UserKey)
-			require.EqualValues(t, []byte{}, it.Value())
+			kv = it.SeekGE([]byte{}, base.SeekGEFlagsNone)
+			require.NotNil(t, kv)
+			require.EqualValues(t, []byte{}, kv.K.UserKey)
+			require.EqualValues(t, []byte{}, mustGetValue(t, kv.V))
 
 			// Add to empty list.
 			err = add(makeIntKey(2), makeValue(2))
 			require.Nil(t, err)
-			require.True(t, it.SeekGE(makeKey("00002"), base.SeekGEFlagsNone))
-			require.EqualValues(t, "00002", it.Key().UserKey)
-			require.EqualValues(t, makeValue(2), it.Value())
+			kv = it.SeekGE(makeKey("00002"), base.SeekGEFlagsNone)
+			require.NotNil(t, kv)
+			require.EqualValues(t, "00002", kv.K.UserKey)
+			require.EqualValues(t, makeValue(2), mustGetValue(t, kv.V))
 
 			// Add first element in non-empty list.
 			err = add(makeIntKey(1), makeValue(1))
 			require.Nil(t, err)
-			require.True(t, it.SeekGE(makeKey("00001"), base.SeekGEFlagsNone))
-			require.EqualValues(t, "00001", it.Key().UserKey)
-			require.EqualValues(t, makeValue(1), it.Value())
+			kv = it.SeekGE(makeKey("00001"), base.SeekGEFlagsNone)
+			require.NotNil(t, kv)
+			require.EqualValues(t, "00001", kv.K.UserKey)
+			require.EqualValues(t, makeValue(1), mustGetValue(t, kv.V))
 
 			// Add last element in non-empty list.
 			err = add(makeIntKey(4), makeValue(4))
 			require.Nil(t, err)
-			require.True(t, it.SeekGE(makeKey("00004"), base.SeekGEFlagsNone))
-			require.EqualValues(t, "00004", it.Key().UserKey)
-			require.EqualValues(t, makeValue(4), it.Value())
+			kv = it.SeekGE(makeKey("00004"), base.SeekGEFlagsNone)
+			require.NotNil(t, kv)
+			require.EqualValues(t, "00004", kv.K.UserKey)
+			require.EqualValues(t, makeValue(4), mustGetValue(t, kv.V))
 
 			// Add element in middle of list.
 			err = add(makeIntKey(3), makeValue(3))
 			require.Nil(t, err)
-			require.True(t, it.SeekGE(makeKey("00003"), base.SeekGEFlagsNone))
-			require.EqualValues(t, "00003", it.Key().UserKey)
-			require.EqualValues(t, makeValue(3), it.Value())
+			kv = it.SeekGE(makeKey("00003"), base.SeekGEFlagsNone)
+			require.NotNil(t, kv)
+			require.EqualValues(t, "00003", kv.K.UserKey)
+			require.EqualValues(t, makeValue(3), mustGetValue(t, kv.V))
 
 			// Try to add element that already exists.
 			err = add(makeIntKey(2), nil)
 			require.Equal(t, ErrRecordExists, err)
-			require.EqualValues(t, "00003", it.Key().UserKey)
-			require.EqualValues(t, makeValue(3), it.Value())
 
 			require.Equal(t, 5, length(l))
 			require.Equal(t, 5, lengthRev(l))
@@ -444,7 +383,7 @@ func TestConcurrentAdd(t *testing.T) {
 
 			for f := 0; f < 2; f++ {
 				go func(f int) {
-					it := newIterAdapter(l.NewIter(nil, nil))
+					it := l.NewIter(nil, nil)
 					add := l.Add
 					if inserter {
 						add = makeInserterAdd(l)
@@ -455,8 +394,9 @@ func TestConcurrentAdd(t *testing.T) {
 
 						key := makeIntKey(i)
 						if add(key, nil) == nil {
-							require.True(t, it.SeekGE(key.UserKey, base.SeekGEFlagsNone))
-							require.EqualValues(t, key, it.Key())
+							kv := it.SeekGE(key.UserKey, base.SeekGEFlagsNone)
+							require.NotNil(t, kv)
+							require.EqualValues(t, key, kv.K)
 						}
 
 						end[i].Done()
@@ -479,61 +419,69 @@ func TestConcurrentAdd(t *testing.T) {
 func TestIteratorNext(t *testing.T) {
 	const n = 100
 	l := NewSkiplist(newArena(arenaSize), bytes.Compare)
-	it := newIterAdapter(l.NewIter(nil, nil))
-
-	require.False(t, it.Valid())
-
-	it.First()
-	require.False(t, it.Valid())
+	it := l.NewIter(nil, nil)
 
+	require.Nil(t, it.First())
 	for i := n - 1; i >= 0; i-- {
 		l.Add(makeIntKey(i), makeValue(i))
 	}
 
-	it.First()
+	kv := it.First()
 	for i := 0; i < n; i++ {
-		require.True(t, it.Valid())
-		require.EqualValues(t, makeIntKey(i), it.Key())
-		require.EqualValues(t, makeValue(i), it.Value())
-		it.Next()
+		require.NotNil(t, kv)
+		require.EqualValues(t, makeIntKey(i), kv.K)
+		require.EqualValues(t, makeValue(i), mustGetValue(t, kv.V))
+		kv = it.Next()
 	}
-	require.False(t, it.Valid())
+	require.Nil(t, kv)
 }
 
 // TestIteratorPrev tests a basic iteration over all nodes from the end.
 func TestIteratorPrev(t *testing.T) {
 	const n = 100
 	l := NewSkiplist(newArena(arenaSize), bytes.Compare)
-	it := newIterAdapter(l.NewIter(nil, nil))
-
-	require.False(t, it.Valid())
-
-	it.Last()
-	require.False(t, it.Valid())
+	it := l.NewIter(nil, nil)
 
+	require.Nil(t, it.Last())
 	var ins Inserter
 	for i := 0; i < n; i++ {
 		ins.Add(l, makeIntKey(i), makeValue(i))
 	}
 
-	it.Last()
+	kv := it.Last()
 	for i := n - 1; i >= 0; i-- {
-		require.True(t, it.Valid())
-		require.EqualValues(t, makeIntKey(i), it.Key())
-		require.EqualValues(t, makeValue(i), it.Value())
-		it.Prev()
+		require.NotNil(t, kv)
+		require.EqualValues(t, makeIntKey(i), kv.K)
+		require.EqualValues(t, makeValue(i), mustGetValue(t, kv.V))
+		kv = it.Prev()
 	}
-	require.False(t, it.Valid())
+	require.Nil(t, kv)
+}
+
+func mustSeekGEKV(t *testing.T, it *Iterator, seekKey []byte, flags base.SeekGEFlags, k, v string) {
+	kv := it.SeekGE(seekKey, flags)
+	require.NotNil(t, kv)
+	require.EqualValues(t, k, string(kv.K.UserKey))
+	gotV := mustGetValue(t, kv.V)
+	require.EqualValues(t, v, string(gotV))
+}
+
+func mustSeekPrefixGEKV(
+	t *testing.T, it *Iterator, seekKey []byte, flags base.SeekGEFlags, k, v string,
+) {
+	kv := it.SeekPrefixGE(seekKey, seekKey, flags)
+	require.NotNil(t, kv)
+	require.EqualValues(t, k, string(kv.K.UserKey))
+	gotV := mustGetValue(t, kv.V)
+	require.EqualValues(t, v, string(gotV))
 }
 
 func TestIteratorSeekGEAndSeekPrefixGE(t *testing.T) {
 	const n = 100
 	l := NewSkiplist(newArena(arenaSize), bytes.Compare)
-	it := newIterAdapter(l.NewIter(nil, nil))
+	it := l.NewIter(nil, nil)
 
-	require.False(t, it.Valid())
-	it.First()
-	require.False(t, it.Valid())
+	require.Nil(t, it.First())
 	// 1000, 1010, 1020, ..., 1990.
 
 	var ins Inserter
@@ -542,124 +490,63 @@ func TestIteratorSeekGEAndSeekPrefixGE(t *testing.T) {
 		ins.Add(l, makeIntKey(v), makeValue(v))
 	}
 
-	require.True(t, it.SeekGE(makeKey(""), base.SeekGEFlagsNone))
-	require.True(t, it.Valid())
-	require.EqualValues(t, "01000", it.Key().UserKey)
-	require.EqualValues(t, "v01000", it.Value())
-
-	require.True(t, it.SeekGE(makeKey("01000"), base.SeekGEFlagsNone))
-	require.True(t, it.Valid())
-	require.EqualValues(t, "01000", it.Key().UserKey)
-	require.EqualValues(t, "v01000", it.Value())
-
-	require.True(t, it.SeekGE(makeKey("01005"), base.SeekGEFlagsNone))
-	require.True(t, it.Valid())
-	require.EqualValues(t, "01010", it.Key().UserKey)
-	require.EqualValues(t, "v01010", it.Value())
-
-	require.True(t, it.SeekGE(makeKey("01010"), base.SeekGEFlagsNone))
-	require.True(t, it.Valid())
-	require.EqualValues(t, "01010", it.Key().UserKey)
-	require.EqualValues(t, "v01010", it.Value())
-
-	require.False(t, it.SeekGE(makeKey("99999"), base.SeekGEFlagsNone))
-	require.False(t, it.Valid())
+	mustSeekGEKV(t, it, makeKey(""), base.SeekGEFlagsNone, "01000", "v01000")
+	mustSeekGEKV(t, it, makeKey("01000"), base.SeekGEFlagsNone, "01000", "v01000")
+	mustSeekGEKV(t, it, makeKey("01005"), base.SeekGEFlagsNone, "01010", "v01010")
+	mustSeekGEKV(t, it, makeKey("01010"), base.SeekGEFlagsNone, "01010", "v01010")
+	require.Nil(t, it.SeekGE(makeKey("99999"), base.SeekGEFlagsNone))
 
 	// Test SeekGE with trySeekUsingNext optimization.
 	{
-		require.True(t, it.SeekGE(makeKey("01000"), base.SeekGEFlagsNone))
-		require.True(t, it.Valid())
-		require.EqualValues(t, "01000", it.Key().UserKey)
-		require.EqualValues(t, "v01000", it.Value())
+		mustSeekGEKV(t, it, makeKey("01000"), base.SeekGEFlagsNone, "01000", "v01000")
 
 		// Seeking to the same key.
-		require.True(t, it.SeekGE(makeKey("01000"), base.SeekGEFlagsNone.EnableTrySeekUsingNext()))
-		require.True(t, it.Valid())
-		require.EqualValues(t, "01000", it.Key().UserKey)
-		require.EqualValues(t, "v01000", it.Value())
+		mustSeekGEKV(t, it, makeKey("01000"), base.SeekGEFlagsNone.EnableTrySeekUsingNext(), "01000", "v01000")
 
 		// Seeking to a nearby key that can be reached using Next.
-		require.True(t, it.SeekGE(makeKey("01020"), base.SeekGEFlagsNone.EnableTrySeekUsingNext()))
-		require.True(t, it.Valid())
-		require.EqualValues(t, "01020", it.Key().UserKey)
-		require.EqualValues(t, "v01020", it.Value())
+		mustSeekGEKV(t, it, makeKey("01020"), base.SeekGEFlagsNone.EnableTrySeekUsingNext(), "01020", "v01020")
 
 		// Seeking to a key that cannot be reached using Next.
-		require.True(t, it.SeekGE(makeKey("01200"), base.SeekGEFlagsNone.EnableTrySeekUsingNext()))
-		require.True(t, it.Valid())
-		require.EqualValues(t, "01200", it.Key().UserKey)
-		require.EqualValues(t, "v01200", it.Value())
+		mustSeekGEKV(t, it, makeKey("01200"), base.SeekGEFlagsNone.EnableTrySeekUsingNext(), "01200", "v01200")
 
 		// Seeking to an earlier key, but the caller lies. Incorrect result.
-		require.True(t, it.SeekGE(makeKey("01100"), base.SeekGEFlagsNone.EnableTrySeekUsingNext()))
-		require.True(t, it.Valid())
-		require.EqualValues(t, "01200", it.Key().UserKey)
-		require.EqualValues(t, "v01200", it.Value())
+		mustSeekGEKV(t, it, makeKey("01100"), base.SeekGEFlagsNone.EnableTrySeekUsingNext(), "01200", "v01200")
 
 		// Telling the truth works.
-		require.True(t, it.SeekGE(makeKey("01100"), base.SeekGEFlagsNone))
-		require.True(t, it.Valid())
-		require.EqualValues(t, "01100", it.Key().UserKey)
-		require.EqualValues(t, "v01100", it.Value())
+		mustSeekGEKV(t, it, makeKey("01100"), base.SeekGEFlagsNone, "01100", "v01100")
 	}
 
 	// Test SeekPrefixGE with trySeekUsingNext optimization.
 	{
-		require.True(t, it.SeekPrefixGE(makeKey("01000"), makeKey("01000"), base.SeekGEFlagsNone))
-		require.True(t, it.Valid())
-		require.EqualValues(t, "01000", it.Key().UserKey)
-		require.EqualValues(t, "v01000", it.Value())
+		mustSeekPrefixGEKV(t, it, makeKey("01000"), base.SeekGEFlagsNone, "01000", "v01000")
 
 		// Seeking to the same key.
-		require.True(t, it.SeekPrefixGE(makeKey("01000"), makeKey("01000"), base.SeekGEFlagsNone.EnableTrySeekUsingNext()))
-		require.True(t, it.Valid())
-		require.EqualValues(t, "01000", it.Key().UserKey)
-		require.EqualValues(t, "v01000", it.Value())
+		mustSeekPrefixGEKV(t, it, makeKey("01000"), base.SeekGEFlagsNone.EnableTrySeekUsingNext(), "01000", "v01000")
 
 		// Seeking to a nearby key that can be reached using Next.
-		require.True(t, it.SeekPrefixGE(makeKey("01020"), makeKey("01020"), base.SeekGEFlagsNone.EnableTrySeekUsingNext()))
-		require.True(t, it.Valid())
-		require.EqualValues(t, "01020", it.Key().UserKey)
-		require.EqualValues(t, "v01020", it.Value())
+		mustSeekPrefixGEKV(t, it, makeKey("01020"), base.SeekGEFlagsNone.EnableTrySeekUsingNext(), "01020", "v01020")
 
 		// Seeking to a key that cannot be reached using Next.
-		require.True(t, it.SeekPrefixGE(makeKey("01200"), makeKey("01200"), base.SeekGEFlagsNone.EnableTrySeekUsingNext()))
-		require.True(t, it.Valid())
-		require.EqualValues(t, "01200", it.Key().UserKey)
-		require.EqualValues(t, "v01200", it.Value())
+		mustSeekPrefixGEKV(t, it, makeKey("01200"), base.SeekGEFlagsNone.EnableTrySeekUsingNext(), "01200", "v01200")
 
 		// Seeking to an earlier key, but the caller lies. Incorrect result.
-		require.True(t, it.SeekPrefixGE(makeKey("01100"), makeKey("01100"), base.SeekGEFlagsNone.EnableTrySeekUsingNext()))
-		require.True(t, it.Valid())
-		require.EqualValues(t, "01200", it.Key().UserKey)
-		require.EqualValues(t, "v01200", it.Value())
+		mustSeekPrefixGEKV(t, it, makeKey("01100"), base.SeekGEFlagsNone.EnableTrySeekUsingNext(), "01200", "v01200")
 
 		// Telling the truth works.
-		require.True(t, it.SeekPrefixGE(makeKey("01100"), makeKey("01100"), base.SeekGEFlagsNone))
-		require.True(t, it.Valid())
-		require.EqualValues(t, "01100", it.Key().UserKey)
-		require.EqualValues(t, "v01100", it.Value())
+		mustSeekPrefixGEKV(t, it, makeKey("01100"), base.SeekGEFlagsNone, "01100", "v01100")
 	}
 
 	// Test seek for empty key.
 	ins.Add(l, base.InternalKey{}, nil)
-	require.True(t, it.SeekGE([]byte{}, base.SeekGEFlagsNone))
-	require.True(t, it.Valid())
-	require.EqualValues(t, "", it.Key().UserKey)
-
-	require.True(t, it.SeekGE(makeKey(""), base.SeekGEFlagsNone))
-	require.True(t, it.Valid())
-	require.EqualValues(t, "", it.Key().UserKey)
+	mustSeekGEKV(t, it, makeKey(""), base.SeekGEFlagsNone, "", "")
 }
 
 func TestIteratorSeekLT(t *testing.T) {
 	const n = 100
 	l := NewSkiplist(newArena(arenaSize), bytes.Compare)
-	it := newIterAdapter(l.NewIter(nil, nil))
+	it := l.NewIter(nil, nil)
 
-	require.False(t, it.Valid())
-	it.First()
-	require.False(t, it.Valid())
+	require.Nil(t, it.First())
 	// 1000, 1010, 1020, ..., 1990.
 	var ins Inserter
 	for i := n - 1; i >= 0; i-- {
@@ -667,40 +554,36 @@ func TestIteratorSeekLT(t *testing.T) {
 		ins.Add(l, makeIntKey(v), makeValue(v))
 	}
 
-	require.False(t, it.SeekLT(makeKey(""), base.SeekLTFlagsNone))
-	require.False(t, it.Valid())
-
-	require.False(t, it.SeekLT(makeKey("01000"), base.SeekLTFlagsNone))
-	require.False(t, it.Valid())
+	require.Nil(t, it.SeekLT(makeKey(""), base.SeekLTFlagsNone))
+	require.Nil(t, it.SeekLT(makeKey("01000"), base.SeekLTFlagsNone))
 
-	require.True(t, it.SeekLT(makeKey("01001"), base.SeekLTFlagsNone))
-	require.True(t, it.Valid())
-	require.EqualValues(t, "01000", it.Key().UserKey)
-	require.EqualValues(t, "v01000", it.Value())
+	kv := it.SeekLT(makeKey("01001"), base.SeekLTFlagsNone)
+	require.NotNil(t, kv)
+	require.EqualValues(t, "01000", kv.K.UserKey)
+	require.EqualValues(t, "v01000", mustGetValue(t, kv.V))
 
-	require.True(t, it.SeekLT(makeKey("01005"), base.SeekLTFlagsNone))
-	require.True(t, it.Valid())
-	require.EqualValues(t, "01000", it.Key().UserKey)
-	require.EqualValues(t, "v01000", it.Value())
+	kv = it.SeekLT(makeKey("01005"), base.SeekLTFlagsNone)
+	require.NotNil(t, kv)
+	require.EqualValues(t, "01000", kv.K.UserKey)
+	require.EqualValues(t, "v01000", mustGetValue(t, kv.V))
 
-	require.True(t, it.SeekLT(makeKey("01991"), base.SeekLTFlagsNone))
-	require.True(t, it.Valid())
-	require.EqualValues(t, "01990", it.Key().UserKey)
-	require.EqualValues(t, "v01990", it.Value())
+	kv = it.SeekLT(makeKey("01991"), base.SeekLTFlagsNone)
+	require.NotNil(t, kv)
+	require.EqualValues(t, "01990", kv.K.UserKey)
+	require.EqualValues(t, "v01990", mustGetValue(t, kv.V))
 
-	require.True(t, it.SeekLT(makeKey("99999"), base.SeekLTFlagsNone))
-	require.True(t, it.Valid())
-	require.EqualValues(t, "01990", it.Key().UserKey)
-	require.EqualValues(t, "v01990", it.Value())
+	kv = it.SeekLT(makeKey("99999"), base.SeekLTFlagsNone)
+	require.NotNil(t, kv)
+	require.EqualValues(t, "01990", kv.K.UserKey)
+	require.EqualValues(t, "v01990", mustGetValue(t, kv.V))
 
 	// Test seek for empty key.
 	ins.Add(l, base.InternalKey{}, nil)
-	require.False(t, it.SeekLT([]byte{}, base.SeekLTFlagsNone))
-	require.False(t, it.Valid())
+	require.Nil(t, it.SeekLT([]byte{}, base.SeekLTFlagsNone))
 
-	require.True(t, it.SeekLT(makeKey("\x01"), base.SeekLTFlagsNone))
-	require.True(t, it.Valid())
-	require.EqualValues(t, "", it.Key().UserKey)
+	kv = it.SeekLT(makeKey("\x01"), base.SeekLTFlagsNone)
+	require.NotNil(t, kv)
+	require.EqualValues(t, "", kv.K.UserKey)
 }
 
 // TODO(peter): test First and Last.
@@ -714,59 +597,62 @@ func TestIteratorBounds(t *testing.T) {
 		return makeIntKey(i).UserKey
 	}
 
-	it := newIterAdapter(l.NewIter(key(3), key(7)))
+	it := l.NewIter(key(3), key(7))
 
 	// SeekGE within the lower and upper bound succeeds.
 	for i := 3; i <= 6; i++ {
 		k := key(i)
-		require.True(t, it.SeekGE(k, base.SeekGEFlagsNone))
-		require.EqualValues(t, string(k), string(it.Key().UserKey))
+		kv := it.SeekGE(k, base.SeekGEFlagsNone)
+		require.NotNil(t, kv)
+		require.EqualValues(t, string(k), string(kv.K.UserKey))
 	}
 
 	// SeekGE before the lower bound still succeeds (only the upper bound is
 	// checked).
 	for i := 1; i < 3; i++ {
 		k := key(i)
-		require.True(t, it.SeekGE(k, base.SeekGEFlagsNone))
-		require.EqualValues(t, string(k), string(it.Key().UserKey))
+		kv := it.SeekGE(k, base.SeekGEFlagsNone)
+		require.NotNil(t, kv)
+		require.EqualValues(t, string(k), string(kv.K.UserKey))
 	}
 
 	// SeekGE beyond the upper bound fails.
 	for i := 7; i < 10; i++ {
-		require.False(t, it.SeekGE(key(i), base.SeekGEFlagsNone))
+		require.Nil(t, it.SeekGE(key(i), base.SeekGEFlagsNone))
 	}
 
-	require.True(t, it.SeekGE(key(6), base.SeekGEFlagsNone))
-	require.EqualValues(t, "00006", it.Key().UserKey)
-	require.EqualValues(t, "v00006", it.Value())
+	mustSeekGEKV(t, it, key(6), base.SeekGEFlagsNone, "00006", "v00006")
 
 	// Next into the upper bound fails.
-	require.False(t, it.Next())
+	require.Nil(t, it.Next())
 
 	// SeekLT within the lower and upper bound succeeds.
 	for i := 4; i <= 7; i++ {
-		require.True(t, it.SeekLT(key(i), base.SeekLTFlagsNone))
-		require.EqualValues(t, string(key(i-1)), string(it.Key().UserKey))
+		kv := it.SeekLT(key(i), base.SeekLTFlagsNone)
+		require.NotNil(t, kv)
+		require.EqualValues(t, string(key(i-1)), string(kv.K.UserKey))
 	}
 
 	// SeekLT beyond the upper bound still succeeds (only the lower bound is
 	// checked).
 	for i := 8; i < 9; i++ {
-		require.True(t, it.SeekLT(key(8), base.SeekLTFlagsNone))
-		require.EqualValues(t, string(key(i-1)), string(it.Key().UserKey))
+		kv := it.SeekLT(key(8), base.SeekLTFlagsNone)
+		require.NotNil(t, kv)
+		require.EqualValues(t, string(key(i-1)), string(kv.K.UserKey))
 	}
 
 	// SeekLT before the lower bound fails.
 	for i := 1; i < 4; i++ {
-		require.False(t, it.SeekLT(key(i), base.SeekLTFlagsNone))
+		require.Nil(t, it.SeekLT(key(i), base.SeekLTFlagsNone))
 	}
 
-	require.True(t, it.SeekLT(key(4), base.SeekLTFlagsNone))
-	require.EqualValues(t, "00003", it.Key().UserKey)
-	require.EqualValues(t, "v00003", it.Value())
+	kv := it.SeekLT(key(4), base.SeekLTFlagsNone)
+	require.NotNil(t, kv)
+	require.EqualValues(t, "00003", kv.K.UserKey)
+	require.EqualValues(t, "v00003", mustGetValue(t, kv.V))
 
 	// Prev into the lower bound fails.
-	require.False(t, it.Prev())
+	require.Nil(t, it.Prev())
 }
 
 func randomKey(rng *rand.Rand, b []byte) base.InternalKey {
diff --git a/internal/base/comparer_test.go b/internal/base/comparer_test.go
index e16d85ea..31db932b 100644
--- a/internal/base/comparer_test.go
+++ b/internal/base/comparer_test.go
@@ -12,15 +12,11 @@ import (
 	"time"
 )
 
-func TestDefAppendSeparator(t *testing.T) {
+func TestDefaultComparer_Separator(t *testing.T) {
 	testCases := []struct {
 		a, b, want string
 	}{
-		// Examples from the doc comments.
 		{"black", "blue", "blb"},
-		{"green", "", "green"},
-		// Non-empty b values. The C++ Level-DB code calls these separators.
-		{"", "2", ""},
 		{"1", "2", "1"},
 		{"1", "29", "2"},
 		{"13", "19", "14"},
@@ -33,22 +29,37 @@ func TestDefAppendSeparator(t *testing.T) {
 		{"1\xff\xff", "19", "1\xff\xff"},
 		{"1\xff\xff", "2", "1\xff\xff"},
 		{"1\xff\xff", "9", "2"},
-		// Empty b values. The C++ Level-DB code calls these successors.
-		{"", "", ""},
-		{"1", "", "1"},
-		{"11", "", "11"},
-		{"11\xff", "", "11\xff"},
-		{"1\xff", "", "1\xff"},
-		{"1\xff\xff", "", "1\xff\xff"},
-		{"\xff", "", "\xff"},
-		{"\xff\xff", "", "\xff\xff"},
-		{"\xff\xff\xff", "", "\xff\xff\xff"},
 	}
 	for _, tc := range testCases {
 		t.Run("", func(t *testing.T) {
 			got := string(DefaultComparer.Separator(nil, []byte(tc.a), []byte(tc.b)))
 			if got != tc.want {
-				t.Errorf("a, b = %q, %q: got %q, want %q", tc.a, tc.b, got, tc.want)
+				t.Errorf("DefaultComparer.Separator(nil, %q, %q) = %q, want %q", tc.a, tc.b, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestDefaultComparer_Successor(t *testing.T) {
+	testCases := []struct {
+		a, want string
+	}{
+		{"green", "h"},
+		{"", ""},
+		{"1", "2"},
+		{"11", "2"},
+		{"11\xff", "2"},
+		{"1\xff", "2"},
+		{"1\xff\xff", "2"},
+		{"\xff", "\xff"},
+		{"\xff\xff", "\xff\xff"},
+		{"\xff\xff\xff", "\xff\xff\xff"},
+	}
+	for _, tc := range testCases {
+		t.Run("", func(t *testing.T) {
+			got := string(DefaultComparer.Successor(nil, []byte(tc.a)))
+			if got != tc.want {
+				t.Errorf("DefaultComparer.Successor(nil, %q) = %q, want %q", tc.a, got, tc.want)
 			}
 		})
 	}
diff --git a/internal/base/filenames_test.go b/internal/base/filenames_test.go
index df07bb8b..15ead6f6 100644
--- a/internal/base/filenames_test.go
+++ b/internal/base/filenames_test.go
@@ -45,6 +45,10 @@ func TestParseFilename(t *testing.T) {
 		"CURRENT.dbtmp":          false,
 		"CURRENT.123456.dbtmp":   true,
 		"temporary.123456.dbtmp": true,
+		"foo.blob":               false,
+		"000000.blob":            true,
+		"000001.blob":            true,
+		"935203523.blob":         true,
 	}
 	fs := vfs.NewMem()
 	for tc, want := range testCases {
@@ -65,6 +69,7 @@ func TestFilenameRoundTrip(t *testing.T) {
 		FileTypeOptions:  true,
 		FileTypeOldTemp:  true,
 		FileTypeTemp:     true,
+		FileTypeBlob:     true,
 		// NB: Log filenames are created and parsed elsewhere in the wal/
 		// package.
 		// FileTypeLog:      true,
diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go
new file mode 100644
index 00000000..ef19a032
--- /dev/null
+++ b/internal/cache/cache_test.go
@@ -0,0 +1,275 @@
+// Copyright 2025 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package cache
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"math/rand/v2"
+	"os"
+	"runtime"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCache(t *testing.T) {
+	// Test data was generated from the python code
+	f, err := os.Open("testdata/cache")
+	require.NoError(t, err)
+
+	cache := newCache(200, 1)
+	defer cache.Unref()
+	h := cache.NewHandle()
+	defer h.Close()
+
+	scanner := bufio.NewScanner(f)
+	line := 1
+
+	for scanner.Scan() {
+		fields := bytes.Fields(scanner.Bytes())
+
+		key, err := strconv.Atoi(string(fields[0]))
+		require.NoError(t, err)
+
+		wantHit := fields[1][0] == 'h'
+
+		var hit bool
+		cv := h.Get(base.DiskFileNum(key), 0)
+		if cv == nil {
+			cv = Alloc(1)
+			cv.RawBuffer()[0] = fields[0][0]
+			h.Set(base.DiskFileNum(key), 0, cv)
+		} else {
+			hit = true
+			if v := cv.RawBuffer(); !bytes.Equal(v, fields[0][:1]) {
+				t.Errorf("%d: cache returned bad data: got %s , want %s\n", line, v, fields[0][:1])
+			}
+		}
+		cv.Release()
+		if hit != wantHit {
+			t.Errorf("%d: cache hit mismatch: got %v, want %v\n", line, hit, wantHit)
+		}
+		line++
+	}
+}
+
+func setTestValue(c *Handle, fileNum base.DiskFileNum, offset uint64, s string, repeat int) {
+	b := bytes.Repeat([]byte(s), repeat)
+	v := Alloc(len(b))
+	copy(v.RawBuffer(), b)
+	c.Set(fileNum, offset, v)
+	v.Release()
+}
+
+func TestCacheDelete(t *testing.T) {
+	cache := newCache(100, 1)
+	defer cache.Unref()
+	h := cache.NewHandle()
+	defer h.Close()
+
+	setTestValue(h, 0, 0, "a", 5)
+	setTestValue(h, 1, 0, "a", 5)
+	setTestValue(h, 2, 0, "a", 5)
+	if expected, size := int64(15), cache.Size(); expected != size {
+		t.Fatalf("expected cache size %d, but found %d", expected, size)
+	}
+	h.Delete(base.DiskFileNum(1), 0)
+	if expected, size := int64(10), cache.Size(); expected != size {
+		t.Fatalf("expected cache size %d, but found %d", expected, size)
+	}
+	if v := h.Get(base.DiskFileNum(0), 0); v == nil {
+		t.Fatalf("expected to find block 0/0")
+	} else {
+		v.Release()
+	}
+	if v := h.Get(base.DiskFileNum(1), 0); v != nil {
+		t.Fatalf("expected to not find block 1/0")
+	}
+	// Deleting a non-existing block does nothing.
+	h.Delete(base.DiskFileNum(1), 0)
+	if expected, size := int64(10), cache.Size(); expected != size {
+		t.Fatalf("expected cache size %d, but found %d", expected, size)
+	}
+}
+
+func TestEvictFile(t *testing.T) {
+	cache := newCache(100, 1)
+	defer cache.Unref()
+	h := cache.NewHandle()
+	defer h.Close()
+
+	setTestValue(h, 0, 0, "a", 5)
+	setTestValue(h, 1, 0, "a", 5)
+	setTestValue(h, 2, 0, "a", 5)
+	setTestValue(h, 2, 1, "a", 5)
+	setTestValue(h, 2, 2, "a", 5)
+	if expected, size := int64(25), cache.Size(); expected != size {
+		t.Fatalf("expected cache size %d, but found %d", expected, size)
+	}
+	h.EvictFile(base.DiskFileNum(0))
+	if expected, size := int64(20), cache.Size(); expected != size {
+		t.Fatalf("expected cache size %d, but found %d", expected, size)
+	}
+	h.EvictFile(base.DiskFileNum(1))
+	if expected, size := int64(15), cache.Size(); expected != size {
+		t.Fatalf("expected cache size %d, but found %d", expected, size)
+	}
+	h.EvictFile(base.DiskFileNum(2))
+	if expected, size := int64(0), cache.Size(); expected != size {
+		t.Fatalf("expected cache size %d, but found %d", expected, size)
+	}
+}
+
+func TestEvictAll(t *testing.T) {
+	// Verify that it is okay to evict all of the data from a cache. Previously
+	// this would trigger a nil-pointer dereference.
+	cache := newCache(100, 1)
+	defer cache.Unref()
+	h := cache.NewHandle()
+	defer h.Close()
+
+	setTestValue(h, 0, 0, "a", 101)
+	setTestValue(h, 1, 0, "a", 101)
+}
+
+func TestMultipleDBs(t *testing.T) {
+	cache := newCache(100, 1)
+	defer cache.Unref()
+	h1 := cache.NewHandle()
+	defer h1.Close()
+	h2 := cache.NewHandle()
+	defer h2.Close()
+
+	setTestValue(h1, 0, 0, "a", 5)
+	setTestValue(h2, 0, 0, "b", 5)
+	if expected, size := int64(10), cache.Size(); expected != size {
+		t.Fatalf("expected cache size %d, but found %d", expected, size)
+	}
+	h1.EvictFile(base.DiskFileNum(0))
+	if expected, size := int64(5), cache.Size(); expected != size {
+		t.Fatalf("expected cache size %d, but found %d", expected, size)
+	}
+	v := h1.Get(base.DiskFileNum(0), 0)
+	if v != nil {
+		t.Fatalf("expected not present, but found %#v", v)
+	}
+	v = h2.Get(base.DiskFileNum(0), 0)
+	if v := v.RawBuffer(); string(v) != "bbbbb" {
+		t.Fatalf("expected bbbbb, but found %s", v)
+	}
+	v.Release()
+}
+
+func TestZeroSize(t *testing.T) {
+	c := New(0)
+	defer c.Unref()
+	h := c.NewHandle()
+	defer h.Close()
+	setTestValue(h, 0, 0, "a", 5)
+}
+
+func TestReserve(t *testing.T) {
+	cache := newCache(4, 2)
+	defer cache.Unref()
+	h1 := cache.NewHandle()
+	defer h1.Close()
+	h2 := cache.NewHandle()
+	defer h2.Close()
+	h3 := cache.NewHandle()
+	defer h3.Close()
+	h4 := cache.NewHandle()
+	defer h4.Close()
+
+	setTestValue(h1, 0, 0, "a", 1)
+	setTestValue(h2, 0, 0, "a", 1)
+	require.EqualValues(t, 2, cache.Size())
+	r := cache.Reserve(1)
+	require.EqualValues(t, 0, cache.Size())
+
+	setTestValue(h1, 0, 0, "a", 1)
+	setTestValue(h2, 0, 0, "a", 1)
+	setTestValue(h3, 0, 0, "a", 1)
+	setTestValue(h4, 0, 0, "a", 1)
+	require.EqualValues(t, 2, cache.Size())
+	r()
+	require.EqualValues(t, 2, cache.Size())
+	setTestValue(h1, 0, 0, "a", 1)
+	setTestValue(h2, 0, 0, "a", 1)
+	require.EqualValues(t, 4, cache.Size())
+}
+
+func TestReserveDoubleRelease(t *testing.T) {
+	cache := newCache(100, 1)
+	defer cache.Unref()
+
+	r := cache.Reserve(10)
+	r()
+
+	result := func() (result string) {
+		defer func() {
+			if v := recover(); v != nil {
+				result = fmt.Sprint(v)
+			}
+		}()
+		r()
+		return ""
+	}()
+	const expected = "pebble: cache reservation already released"
+	if expected != result {
+		t.Fatalf("expected %q, but found %q", expected, result)
+	}
+}
+
+func TestCacheStressSetExisting(t *testing.T) {
+	cache := newCache(1, 1)
+	defer cache.Unref()
+	h := cache.NewHandle()
+	defer h.Close()
+
+	var wg sync.WaitGroup
+	for i := 0; i < 10; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			for j := 0; j < 10000; j++ {
+				setTestValue(h, 0, uint64(i), "a", 1)
+				runtime.Gosched()
+			}
+		}(i)
+	}
+	wg.Wait()
+}
+
+func BenchmarkCacheGet(b *testing.B) {
+	const size = 100000
+
+	cache := newCache(size, 1)
+	defer cache.Unref()
+	h := cache.NewHandle()
+	defer h.Close()
+
+	for i := 0; i < size; i++ {
+		setTestValue(h, 0, 0, "a", 1)
+	}
+
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		rng := rand.New(rand.NewPCG(0, uint64(time.Now().UnixNano())))
+
+		for pb.Next() {
+			v := h.Get(base.DiskFileNum(0), uint64(rng.IntN(size)))
+			if v == nil {
+				b.Fatal("failed to lookup value")
+			}
+			v.Release()
+		}
+	})
+}
diff --git a/internal/cache/clockpro_test.go b/internal/cache/clockpro_test.go
index 9f5c2165..fc653d1e 100644
--- a/internal/cache/clockpro_test.go
+++ b/internal/cache/clockpro_test.go
@@ -1,265 +1,34 @@
-// Copyright 2018. All rights reserved. Use of this source code is governed by
-// an MIT-style license that can be found in the LICENSE file.
+// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
 
 package cache
 
 import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"math/rand/v2"
-	"os"
-	"runtime"
-	"strconv"
-	"sync"
 	"testing"
-	"time"
 
-	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/stretchr/testify/require"
 )
 
-func TestCache(t *testing.T) {
-	// Test data was generated from the python code
-	f, err := os.Open("testdata/cache")
-	require.NoError(t, err)
-
-	cache := newShards(200, 1)
-	defer cache.Unref()
-
-	scanner := bufio.NewScanner(f)
-	line := 1
-
-	for scanner.Scan() {
-		fields := bytes.Fields(scanner.Bytes())
-
-		key, err := strconv.Atoi(string(fields[0]))
-		require.NoError(t, err)
-
-		wantHit := fields[1][0] == 'h'
-
-		var hit bool
-		cv := cache.Get(1, base.DiskFileNum(key), 0)
-		if cv == nil {
-			cv = Alloc(1)
-			cv.RawBuffer()[0] = fields[0][0]
-			cache.Set(1, base.DiskFileNum(key), 0, cv)
-		} else {
-			hit = true
-			if v := cv.RawBuffer(); !bytes.Equal(v, fields[0][:1]) {
-				t.Errorf("%d: cache returned bad data: got %s , want %s\n", line, v, fields[0][:1])
-			}
-		}
-		cv.Release()
-		if hit != wantHit {
-			t.Errorf("%d: cache hit mismatch: got %v, want %v\n", line, hit, wantHit)
-		}
-		line++
-	}
-}
-
-func setTestValue(
-	cache *Cache, id ID, fileNum base.DiskFileNum, offset uint64, s string, repeat int,
-) {
-	b := bytes.Repeat([]byte(s), repeat)
-	v := Alloc(len(b))
-	copy(v.RawBuffer(), b)
-	cache.Set(id, fileNum, offset, v)
-	v.Release()
-}
-
-func TestCacheDelete(t *testing.T) {
-	cache := newShards(100, 1)
-	defer cache.Unref()
-
-	setTestValue(cache, 1, 0, 0, "a", 5)
-	setTestValue(cache, 1, 1, 0, "a", 5)
-	setTestValue(cache, 1, 2, 0, "a", 5)
-	if expected, size := int64(15), cache.Size(); expected != size {
-		t.Fatalf("expected cache size %d, but found %d", expected, size)
-	}
-	cache.Delete(1, base.DiskFileNum(1), 0)
-	if expected, size := int64(10), cache.Size(); expected != size {
-		t.Fatalf("expected cache size %d, but found %d", expected, size)
-	}
-	if v := cache.Get(1, base.DiskFileNum(0), 0); v == nil {
-		t.Fatalf("expected to find block 0/0")
-	} else {
-		v.Release()
-	}
-	if v := cache.Get(1, base.DiskFileNum(1), 0); v != nil {
-		t.Fatalf("expected to not find block 1/0")
-	}
-	// Deleting a non-existing block does nothing.
-	cache.Delete(1, base.DiskFileNum(1), 0)
-	if expected, size := int64(10), cache.Size(); expected != size {
-		t.Fatalf("expected cache size %d, but found %d", expected, size)
-	}
-}
-
-func TestEvictFile(t *testing.T) {
-	cache := newShards(100, 1)
-	defer cache.Unref()
-
-	setTestValue(cache, 1, 0, 0, "a", 5)
-	setTestValue(cache, 1, 1, 0, "a", 5)
-	setTestValue(cache, 1, 2, 0, "a", 5)
-	setTestValue(cache, 1, 2, 1, "a", 5)
-	setTestValue(cache, 1, 2, 2, "a", 5)
-	if expected, size := int64(25), cache.Size(); expected != size {
-		t.Fatalf("expected cache size %d, but found %d", expected, size)
-	}
-	cache.EvictFile(1, base.DiskFileNum(0))
-	if expected, size := int64(20), cache.Size(); expected != size {
-		t.Fatalf("expected cache size %d, but found %d", expected, size)
-	}
-	cache.EvictFile(1, base.DiskFileNum(1))
-	if expected, size := int64(15), cache.Size(); expected != size {
-		t.Fatalf("expected cache size %d, but found %d", expected, size)
-	}
-	cache.EvictFile(1, base.DiskFileNum(2))
-	if expected, size := int64(0), cache.Size(); expected != size {
-		t.Fatalf("expected cache size %d, but found %d", expected, size)
-	}
-}
-
-func TestEvictAll(t *testing.T) {
-	// Verify that it is okay to evict all of the data from a cache. Previously
-	// this would trigger a nil-pointer dereference.
-	cache := newShards(100, 1)
-	defer cache.Unref()
-
-	setTestValue(cache, 1, 0, 0, "a", 101)
-	setTestValue(cache, 1, 1, 0, "a", 101)
-}
-
-func TestMultipleDBs(t *testing.T) {
-	cache := newShards(100, 1)
-	defer cache.Unref()
-
-	setTestValue(cache, 1, 0, 0, "a", 5)
-	setTestValue(cache, 2, 0, 0, "b", 5)
-	if expected, size := int64(10), cache.Size(); expected != size {
-		t.Fatalf("expected cache size %d, but found %d", expected, size)
-	}
-	cache.EvictFile(1, base.DiskFileNum(0))
-	if expected, size := int64(5), cache.Size(); expected != size {
-		t.Fatalf("expected cache size %d, but found %d", expected, size)
-	}
-	v := cache.Get(1, base.DiskFileNum(0), 0)
-	if v != nil {
-		t.Fatalf("expected not present, but found %#v", v)
-	}
-	v = cache.Get(2, base.DiskFileNum(0), 0)
-	if v := v.RawBuffer(); string(v) != "bbbbb" {
-		t.Fatalf("expected bbbbb, but found %s", v)
-	}
-	v.Release()
-}
-
-func TestZeroSize(t *testing.T) {
-	cache := newShards(0, 1)
-	defer cache.Unref()
-
-	setTestValue(cache, 1, 0, 0, "a", 5)
-}
-
-func TestReserve(t *testing.T) {
-	cache := newShards(4, 2)
-	defer cache.Unref()
-
-	setTestValue(cache, 1, 0, 0, "a", 1)
-	setTestValue(cache, 2, 0, 0, "a", 1)
-	require.EqualValues(t, 2, cache.Size())
-	r := cache.Reserve(1)
-	require.EqualValues(t, 0, cache.Size())
-
-	setTestValue(cache, 1, 0, 0, "a", 1)
-	setTestValue(cache, 2, 0, 0, "a", 1)
-	setTestValue(cache, 3, 0, 0, "a", 1)
-	setTestValue(cache, 4, 0, 0, "a", 1)
-	require.EqualValues(t, 2, cache.Size())
-	r()
-	require.EqualValues(t, 2, cache.Size())
-	setTestValue(cache, 1, 0, 0, "a", 1)
-	setTestValue(cache, 2, 0, 0, "a", 1)
-	require.EqualValues(t, 4, cache.Size())
-}
-
-func TestReserveDoubleRelease(t *testing.T) {
-	cache := newShards(100, 1)
-	defer cache.Unref()
-
-	r := cache.Reserve(10)
-	r()
-
-	result := func() (result string) {
-		defer func() {
-			if v := recover(); v != nil {
-				result = fmt.Sprint(v)
-			}
-		}()
-		r()
-		return ""
-	}()
-	const expected = "pebble: cache reservation already released"
-	if expected != result {
-		t.Fatalf("expected %q, but found %q", expected, result)
-	}
-}
-
-func TestCacheStressSetExisting(t *testing.T) {
-	cache := newShards(1, 1)
-	defer cache.Unref()
-
-	var wg sync.WaitGroup
-	for i := 0; i < 10; i++ {
-		wg.Add(1)
-		go func(i int) {
-			defer wg.Done()
-			for j := 0; j < 10000; j++ {
-				setTestValue(cache, 1, 0, uint64(i), "a", 1)
-				runtime.Gosched()
-			}
-		}(i)
-	}
-	wg.Wait()
-}
-
-func BenchmarkCacheGet(b *testing.B) {
-	const size = 100000
-
-	cache := newShards(size, 1)
-	defer cache.Unref()
-
-	for i := 0; i < size; i++ {
-		setTestValue(cache, 1, 0, 0, "a", 1)
-	}
-
-	b.ResetTimer()
-	b.RunParallel(func(pb *testing.PB) {
-		rng := rand.New(rand.NewPCG(0, uint64(time.Now().UnixNano())))
-
-		for pb.Next() {
-			v := cache.Get(1, base.DiskFileNum(0), uint64(rng.IntN(size)))
-			if v == nil {
-				b.Fatal("failed to lookup value")
-			}
-			v.Release()
-		}
-	})
-}
-
 func TestReserveColdTarget(t *testing.T) {
 	// If coldTarget isn't updated when we call shard.Reserve,
 	// then we unnecessarily remove nodes from the
 	// cache.
 
-	cache := newShards(100, 1)
+	cache := newCache(100, 1)
 	defer cache.Unref()
+	h := make([]*Handle, 50)
+	for i := range h {
+		h[i] = cache.NewHandle()
+	}
+	defer func() {
+		for i := range h {
+			h[i].Close()
+		}
+	}()
 
-	for i := 0; i < 50; i++ {
-		setTestValue(cache, ID(i+1), 0, 0, "a", 1)
+	for i := range h {
+		setTestValue(h[i], 0, 0, "a", 1)
 	}
 
 	if cache.Size() != 50 {
diff --git a/internal/cache/read_shard_test.go b/internal/cache/read_shard_test.go
index 3816a491..b146f831 100644
--- a/internal/cache/read_shard_test.go
+++ b/internal/cache/read_shard_test.go
@@ -35,7 +35,7 @@ type testReader struct {
 }
 
 func newTestReader(
-	ctx context.Context, id ID, fileNum base.DiskFileNum, offset uint64, mu *sync.Mutex,
+	ctx context.Context, id handleID, fileNum base.DiskFileNum, offset uint64, mu *sync.Mutex,
 ) *testReader {
 	r := &testReader{
 		ctx: ctx,
@@ -148,7 +148,7 @@ func TestReadShard(t *testing.T) {
 					ctx, cancelFunc = context.WithCancel(ctx)
 					cancelFunc()
 				}
-				r := newTestReader(ctx, ID(id), base.DiskFileNum(fileNum), uint64(offset), &mu)
+				r := newTestReader(ctx, handleID(id), base.DiskFileNum(fileNum), uint64(offset), &mu)
 				val := r.getAsync(c)
 				if val != nil {
 					return fmt.Sprintf("val: %s", *val)
@@ -200,8 +200,7 @@ func TestReadShard(t *testing.T) {
 // testSyncReaders is the config for multiple readers concurrently reading the
 // same block.
 type testSyncReaders struct {
-	// id, fileNum, offset are the key.
-	id      ID
+	handle  *Handle
 	fileNum base.DiskFileNum
 	offset  uint64
 	// val will be the value read, if not found in the cache.
@@ -221,6 +220,19 @@ type testSyncReaders struct {
 func TestReadShardConcurrent(t *testing.T) {
 	cache := New(rand.Int63n(20 << 10))
 	defer cache.Unref()
+	c := cache.NewHandle()
+	defer c.Close()
+
+	handles := make([]*Handle, 50)
+	for i := range handles {
+		handles[i] = cache.NewHandle()
+	}
+	defer func() {
+		for i := range handles {
+			handles[i].Close()
+		}
+	}()
+
 	var differentReaders []*testSyncReaders
 	// 50 blocks are read.
 	for i := 0; i < 50; i++ {
@@ -228,7 +240,7 @@ func TestReadShardConcurrent(t *testing.T) {
 		val := make([]byte, valLen)
 		crand.Read(val)
 		readers := &testSyncReaders{
-			id:                 ID(rand.Uint64()),
+			handle:             handles[rand.Intn(len(handles))],
 			fileNum:            base.DiskFileNum(rand.Uint64()),
 			offset:             rand.Uint64(),
 			val:                val,
@@ -242,7 +254,7 @@ func TestReadShardConcurrent(t *testing.T) {
 	for _, r := range differentReaders {
 		for j := 0; j < r.numReaders; j++ {
 			go func(r *testSyncReaders, index int) {
-				v, rh, _, _, err := cache.GetWithReadHandle(context.Background(), r.id, r.fileNum, r.offset)
+				v, rh, _, _, err := r.handle.GetWithReadHandle(context.Background(), r.fileNum, r.offset)
 				require.NoError(t, err)
 				if v != nil {
 					require.Equal(t, r.val, v.RawBuffer())
diff --git a/internal/manifest/blob_metadata_test.go b/internal/manifest/blob_metadata_test.go
new file mode 100644
index 00000000..1f35dce6
--- /dev/null
+++ b/internal/manifest/blob_metadata_test.go
@@ -0,0 +1,46 @@
+// Copyright 2025 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package manifest
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestBlobFileMetadata_ParseRoundTrip(t *testing.T) {
+	testCases := []struct {
+		name   string
+		input  string
+		output string
+	}{
+		{
+			name:  "verbatim",
+			input: "000001 size:[903530 (882KB)] vals:[39531 (39KB)]",
+		},
+		{
+			name:   "whitespace is insignificant",
+			input:  "000001   size  : [ 903530 (882KB )] vals: [ 39531 ( 39KB ) ]",
+			output: "000001 size:[903530 (882KB)] vals:[39531 (39KB)]",
+		},
+		{
+			name:   "humanized sizes are optional",
+			input:  "000001 size:[903530] vals:[39531]",
+			output: "000001 size:[903530 (882KB)] vals:[39531 (39KB)]",
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			m, err := ParseBlobFileMetadataDebug(tc.input)
+			require.NoError(t, err)
+			got := m.String()
+			want := tc.input
+			if tc.output != "" {
+				want = tc.output
+			}
+			require.Equal(t, want, got)
+		})
+	}
+}
diff --git a/internal/manifest/manifest_test.go b/internal/manifest/manifest_test.go
index 0ba7ad33..cbca5f80 100644
--- a/internal/manifest/manifest_test.go
+++ b/internal/manifest/manifest_test.go
@@ -32,8 +32,6 @@ func TestInuseKeyRangesRandomized(t *testing.T) {
 
 	// Generate a random database by running the metamorphic test.
 	testOpts := metamorphic.RandomOptions(rng, metamorphic.TestkeysKeyFormat, nil /* custom opt parsers */)
-	testOpts.Opts.Cache.Ref()
-	defer testOpts.Opts.Cache.Unref()
 	{
 		nOps := 10000
 		if buildtags.Race {
diff --git a/internal/manifest/testdata/version_edit_decode b/internal/manifest/testdata/version_edit_decode
new file mode 100644
index 00000000..dff340d7
--- /dev/null
+++ b/internal/manifest/testdata/version_edit_decode
@@ -0,0 +1,90 @@
+decode
+01                           # <tagComparator>
+1a                           # len("leveldb.BytewiseComparator") = 26
+"leveldb.BytewiseComparator" # Comparer name
+0302                         # <tagNextFileNumber>, 2
+0400                         # <tagLastSequence>, 0
+0202                         # <tagLogNumber>, 2
+0303                         # <tagNextFileNumber>, 3
+0409                         # <tagLastSequence>, 9
+----
+011a6c6576656c64622e4279746577697365436f6d70617261746f720302
+0400020203030409
+  comparer:     leveldb.BytewiseComparator  log-num:       2
+  next-file-num: 3
+  last-seq-num:  9
+
+decode
+01                           # <tagComparator>
+1a                           # len("leveldb.BytewiseComparator") = 26
+"leveldb.BytewiseComparator" # Comparer name
+0202                         # <tagLogNumber>, 2
+0307                         # <tagNextFileNumber>, 7
+0400                         # <tagLastSequence>, 0
+----
+011a6c6576656c64622e4279746577697365436f6d70617261746f720202
+03070400
+  comparer:     leveldb.BytewiseComparator  log-num:       2
+  next-file-num: 7
+
+decode
+0205                         # <tagLogNumber>, 5
+0306                         # <tagNextFileNumber>, 6
+040e                         # <tagLastSequence>, 14
+67                           # <tagNewFile4>
+00                           #   . Level           = L0
+04                           #   . FileNum         = 000004
+c505                         #   . FileSize        = 709
+0b 626172000e000000000000    #   . Smallest        = "bar"#14,DEL
+0b 666f6f010d000000000000    #   . Largest         = "foo"#13,SET
+0c                           #   . Smallest Seqnum = 12
+0e                           #   . Largest Seqnum  = 14
+06 05 84a6b8ab06             #   . <customTagCreationTime> <len(field)=5> <1701712644>
+01                           #   . <customTagTerminate>
+----
+02050306040e670004c5050b626172000e0000000000000b666f6f010d00
+00000000000c0e060584a6b8ab0601
+  log-num:       5
+  next-file-num: 6
+  last-seq-num:  14
+  add-table:     L0 000004:[bar#14,DEL-foo#13,SET] (2023-12-04T17:57:24Z)
+
+decode
+0219                         # <tagLogNumber>, 19
+0366                         # <tagNextFileNumber>, 42
+048410                       # <tagLastSequence>, 2052
+6b                           # <tagNewBlobFile>
+29                           #   . FileNum      = 000029
+d8a301                       #   . Size         = 20952
+f9a006                       #   . ValueSize    = 102521
+9f9485bd06                   #   . CreationTime = 1738623519
+6c                           # <tagDeleteBlobFile>
+21                           #   . FileNum      = 000021
+----
+021903660484106b29d8a301f9a0069f9485bd066c21
+  log-num:       25
+  next-file-num: 102
+  last-seq-num:  2052
+  add-blob-file: 000041 size:[20952 (20KB)] vals:[102521 (100KB)]
+  del-blob-file: 000033
+
+encode
+  add-table:     L6 000029:[bar#14,DEL-foo#13,SET] blobrefs:[(000041: 20952)]
+  add-blob-file: 000041 size:[20952 (20KB)] vals:[102521 (100KB)]
+  del-blob-file: 000033
+----
+67061d000b626172000e0000000000000b666f6f010d0000000000000000
+450129d8a301016b29d8a301f9a006006c21
+  add-table:     L6 000029:[bar#14,DEL-foo#13,SET] seqnums:[0-0] points:[bar#14,DEL-foo#13,SET] blobrefs:[(000041: 20952)]
+  add-blob-file: 000041 size:[20952 (20KB)] vals:[102521 (100KB)]
+  del-blob-file: 000033
+
+decode
+67061d000b626172000e0000000000000b666f6f010d0000000000000000
+450129d8a301016b29d8a301f9a006006c21
+----
+67061d000b626172000e0000000000000b666f6f010d0000000000000000
+450129d8a301016b29d8a301f9a006006c21
+  add-table:     L6 000029:[bar#14,DEL-foo#13,SET]
+  add-blob-file: 000041 size:[20952 (20KB)] vals:[102521 (100KB)]
+  del-blob-file: 000033
diff --git a/internal/manifest/version_edit_test.go b/internal/manifest/version_edit_test.go
index 615f7a23..d12071f5 100644
--- a/internal/manifest/version_edit_test.go
+++ b/internal/manifest/version_edit_test.go
@@ -6,18 +6,19 @@ package manifest
 
 import (
 	"bytes"
+	"encoding/hex"
 	"fmt"
 	"io"
-	"os"
-	"reflect"
 	"slices"
+	"strconv"
 	"strings"
 	"testing"
+	"unicode"
 
+	"github.com/cockroachdb/crlib/crstrings"
 	"github.com/cockroachdb/datadriven"
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
-	"github.com/cockroachdb/pebble/record"
 	"github.com/cockroachdb/pebble/sstable"
 	"github.com/kr/pretty"
 	"github.com/stretchr/testify/require"
@@ -261,123 +262,73 @@ func TestVersionEditRoundTrip(t *testing.T) {
 }
 
 func TestVersionEditDecode(t *testing.T) {
-	// TODO(radu): these should be datadriven tests that output the encoded and
-	// decoded edits.
-	cmp := base.DefaultComparer.Compare
-	m := (&FileMetadata{
-		FileNum:               4,
-		Size:                  709,
-		SmallestSeqNum:        12,
-		LargestSeqNum:         14,
-		LargestSeqNumAbsolute: 14,
-		CreationTime:          1701712644,
-	}).ExtendPointKeyBounds(
-		cmp,
-		base.MakeInternalKey([]byte("bar"), 14, base.InternalKeyKindDelete),
-		base.MakeInternalKey([]byte("foo"), 13, base.InternalKeyKindSet),
-	)
-	m.InitPhysicalBacking()
-
-	testCases := []struct {
-		filename     string
-		encodedEdits []string
-		edits        []VersionEdit
-	}{
-		// db-stage-1 and db-stage-2 have the same manifest.
-		{
-			filename: "db-stage-1/MANIFEST-000001",
-			encodedEdits: []string{
-				"\x01\x1aleveldb.BytewiseComparator\x03\x02\x04\x00",
-				"\x02\x02\x03\x03\x04\t",
-			},
-			edits: []VersionEdit{
-				{
-					ComparerName: "leveldb.BytewiseComparator",
-					NextFileNum:  2,
-				},
-				{
-					MinUnflushedLogNum: 0x2,
-					NextFileNum:        0x3,
-					LastSeqNum:         0x9,
-				},
-			},
-		},
-		// db-stage-3 and db-stage-4 have the same manifest.
-		{
-			filename: "db-stage-3/MANIFEST-000006",
-			encodedEdits: []string{
-				"\x01\x1aleveldb.BytewiseComparator\x02\x02\x03\a\x04\x00",
-				"\x02\x05\x03\x06\x04\x0eg\x00\x04\xc5\x05\vbar\x00\x0e\x00\x00\x00\x00\x00\x00\vfoo\x01\r\x00\x00\x00\x00\x00\x00\f\x0e\x06\x05\x84\xa6\xb8\xab\x06\x01",
-			},
-			edits: []VersionEdit{
-				{
-					ComparerName:       "leveldb.BytewiseComparator",
-					MinUnflushedLogNum: 0x2,
-					NextFileNum:        0x7,
-				},
-				{
-					MinUnflushedLogNum: 0x5,
-					NextFileNum:        0x6,
-					LastSeqNum:         0xe,
-					NewFiles: []NewFileEntry{
-						{
-							Level: 0,
-							Meta:  m,
-						},
-					},
-				},
-			},
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run("", func(t *testing.T) {
-			f, err := os.Open("../../testdata/" + tc.filename)
-			if err != nil {
-				t.Fatalf("filename=%q: open error: %v", tc.filename, err)
-			}
-			defer f.Close()
-			i, r := 0, record.NewReader(f, 0 /* logNum */)
-			for {
-				rr, err := r.Next()
-				if err == io.EOF {
-					break
-				}
-				if err != nil {
-					t.Fatalf("filename=%q i=%d: record reader error: %v", tc.filename, i, err)
+	var inputBuf, outputBuf bytes.Buffer
+	datadriven.RunTest(t, "testdata/version_edit_decode",
+		func(t *testing.T, d *datadriven.TestData) string {
+			inputBuf.Reset()
+			outputBuf.Reset()
+			switch d.Cmd {
+			case "encode":
+				ve, err := ParseVersionEditDebug(d.Input)
+				require.NoError(t, err)
+				require.NoError(t, ve.Encode(&inputBuf))
+				serialized := inputBuf.Bytes()
+				for len(serialized) > 0 {
+					line := serialized[:min(30, len(serialized))]
+					serialized = serialized[len(line):]
+					outputBuf.WriteString(hex.EncodeToString(line))
+					outputBuf.WriteByte('\n')
 				}
-				if i >= len(tc.edits) {
-					t.Fatalf("filename=%q i=%d: too many version edits", tc.filename, i+1)
+				fmt.Fprint(&outputBuf, ve.DebugString(base.DefaultFormatter))
+				return outputBuf.String()
+
+			case "decode":
+				for _, l := range crstrings.Lines(d.Input) {
+					i := strings.IndexByte(l, '#')
+					if i == -1 {
+						i = len(l)
+					}
+					s := strings.Map(func(r rune) rune {
+						if unicode.IsSpace(r) {
+							return -1
+						}
+						return r
+					}, l[:i])
+					if strings.HasPrefix(s, "\"") {
+						unquoted, err := strconv.Unquote(s)
+						require.NoError(t, err)
+						io.WriteString(&inputBuf, unquoted)
+					} else {
+						b, err := hex.DecodeString(s)
+						require.NoError(t, err)
+						inputBuf.Write(b)
+					}
 				}
 
-				encodedEdit, err := io.ReadAll(rr)
-				if err != nil {
-					t.Fatalf("filename=%q i=%d: read error: %v", tc.filename, i, err)
-				}
-				if s := string(encodedEdit); s != tc.encodedEdits[i] {
-					t.Fatalf("filename=%q i=%d: got encoded %q, want %q", tc.filename, i, s, tc.encodedEdits[i])
+				r := bytes.NewReader(inputBuf.Bytes())
+				var ve VersionEdit
+				if err := ve.Decode(r); err != nil {
+					return fmt.Sprintf("err: %v", err)
 				}
 
-				var edit VersionEdit
-				err = edit.Decode(bytes.NewReader(encodedEdit))
-				if err != nil {
-					t.Fatalf("filename=%q i=%d: decode error: %v", tc.filename, i, err)
-				}
-				if !reflect.DeepEqual(edit, tc.edits[i]) {
-					t.Fatalf("filename=%q i=%d: decode\n\tgot  %#v\n\twant %#v\n%s", tc.filename, i, edit, tc.edits[i],
-						strings.Join(pretty.Diff(edit, tc.edits[i]), "\n"))
-				}
-				if err := checkRoundTrip(edit); err != nil {
-					t.Fatalf("filename=%q i=%d: round trip: %v", tc.filename, i, err)
+				// Ensure the version edit roundtrips.
+				if err := checkRoundTrip(ve); err != nil {
+					t.Fatal(err)
 				}
 
-				i++
-			}
-			if i != len(tc.edits) {
-				t.Fatalf("filename=%q: got %d edits, want %d", tc.filename, i, len(tc.edits))
+				serialized := inputBuf.Bytes()
+				for len(serialized) > 0 {
+					line := serialized[:min(30, len(serialized))]
+					serialized = serialized[len(line):]
+					outputBuf.WriteString(hex.EncodeToString(line))
+					outputBuf.WriteByte('\n')
+				}
+				fmt.Fprint(&outputBuf, ve.String())
+				return outputBuf.String()
+			default:
+				panic(fmt.Sprintf("unknown command: %s", d.Cmd))
 			}
 		})
-	}
 }
 
 func TestVersionEditEncodeLastSeqNum(t *testing.T) {
@@ -513,6 +464,12 @@ func TestParseVersionEditDebugRoundTrip(t *testing.T) {
 		input  string
 		output string
 	}{
+		{
+			input: `  del-blob-file: 000001`,
+		},
+		{
+			input: `  add-blob-file: 000005 size:[20535 (20KB)] vals:[25935 (25KB)]`,
+		},
 		{
 			input: `  add-table:     L1 000001:[a#0,SET-z#0,DEL] seqnums:[0-0] points:[a#0,SET-z#0,DEL] size:1`,
 		},
@@ -531,6 +488,23 @@ func TestParseVersionEditDebugRoundTrip(t *testing.T) {
 				`  add-table:     L2 000002:[a#0,SET-z#0,DEL] seqnums:[0-0] points:[a#0,SET-z#0,DEL] size:2`,
 			}, "\n"),
 		},
+		{
+			input: strings.Join([]string{
+				`  del-table:     L1 000002`,
+				`  del-table:     L3 000003`,
+				`  add-table:     L1 000001:[a#0,SET-z#0,DEL] seqnums:[0-0] points:[a#0,SET-z#0,DEL] size:1`,
+				`  add-table:     L2 000002:[a#0,SET-z#0,DEL] seqnums:[0-0] points:[a#0,SET-z#0,DEL] size:2`,
+				`  add-blob-file: 000005 size:[20535 (20KB)] vals:[25935 (25KB)]`,
+				`  del-blob-file: 000004`,
+				`  del-blob-file: 000006`,
+			}, "\n"),
+		},
+		{
+			input: strings.Join([]string{
+				`  add-table:     L1 000001:[a#0,SET-z#0,DEL] seqnums:[0-0] points:[a#0,SET-z#0,DEL] size:1`,
+				`  add-table:     L2 000002:[a#0,SET-z#0,DEL] seqnums:[0-0] points:[a#0,SET-z#0,DEL] size:2 blobrefs:[(002431: 3008533), (002432: 10534)]`,
+			}, "\n"),
+		},
 	}
 	for _, tc := range testCases {
 		t.Run("", func(t *testing.T) {
diff --git a/internal/manifest/version_test.go b/internal/manifest/version_test.go
index 97045781..c15c0213 100644
--- a/internal/manifest/version_test.go
+++ b/internal/manifest/version_test.go
@@ -414,6 +414,10 @@ func TestFileMetadata_ParseRoundTrip(t *testing.T) {
 			name:  "virtual",
 			input: "000001(000008):[a#0,SET-z#0,DEL] seqnums:[0-0] points:[a#0,SET-z#0,DEL]",
 		},
+		{
+			name:  "blobrefs",
+			input: "000196:[bar#0,SET-foo#0,SET] seqnums:[0-0] points:[bar#0,SET-foo#0,SET] blobrefs:[(000191: 2952), (000075: 108520)]",
+		},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
diff --git a/internal/metamorphic/crossversion/crossversion_test.go b/internal/metamorphic/crossversion/crossversion_test.go
index 7285465e..a4796c23 100644
--- a/internal/metamorphic/crossversion/crossversion_test.go
+++ b/internal/metamorphic/crossversion/crossversion_test.go
@@ -17,6 +17,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -130,18 +131,6 @@ type pebbleVersion struct {
 	TestBinaryPath string
 }
 
-type initialState struct {
-	desc string
-	path string
-}
-
-func (s initialState) String() string {
-	if s.desc == "" {
-		return "<empty>"
-	}
-	return s.desc
-}
-
 func runCrossVersion(
 	ctx context.Context,
 	t *testing.T,
@@ -179,22 +168,22 @@ func runCrossVersion(
 	initialStates := []initialState{{}}
 	for i := range versions {
 		t.Logf("Running tests with version %s with %d initial state(s).", versions[i].SHA, len(initialStates))
-		histories, nextInitialStates, err := runVersion(ctx, t, &fatalOnce, rootDir, versions[i], versionSeeds[i], initialStates)
-		if err != nil {
-			return err
-		}
+		subrunResults := runVersion(ctx, t, &fatalOnce, rootDir, versions[i], versionSeeds[i], initialStates)
 
 		// All the initial states described the same state and all of this
 		// version's metamorphic runs used the same seed, so all of the
 		// resulting histories should be identical.
+		histories := subrunResults.historyPaths()
 		if h, diff := metamorphic.CompareHistories(t, histories); h > 0 {
-			fatalf(t, &fatalOnce, rootDir, "Metamorphic test divergence between %q and %q:\nDiff:\n%s",
-				nextInitialStates[0].desc, nextInitialStates[h].desc, diff)
+			fatalf(t, &fatalOnce, []string{subrunResults[0].runDir, subrunResults[h].runDir},
+				"Metamorphic test divergence between %q and %q:\nDiff:\n%s",
+				subrunResults[0].initialState.desc, subrunResults[h].initialState.desc, diff)
 		}
 
 		// Prune the set of initial states we collected for this version, using
 		// the deterministic randomness of prng to pick which states we keep.
-		if len(nextInitialStates) > factor {
+		nextInitialStates := subrunResults.initialStates()
+		if len(subrunResults) > factor {
 			prng.Shuffle(len(nextInitialStates), func(i, j int) {
 				nextInitialStates[i], nextInitialStates[j] = nextInitialStates[j], nextInitialStates[i]
 			})
@@ -209,6 +198,52 @@ func runCrossVersion(
 	return nil
 }
 
+// subrunResult describes the output of a single run of the metamorphic test
+// against a particular version.
+type subrunResult struct {
+	// runDir is the path to the directory containing the output of this
+	// TestMeta run.
+	runDir string
+	// historyPath is the path to the history file recording the output of this
+	// run. All the histories produced by a single call to runVersion should be
+	// semantically equivalent.
+	historyPath string
+	// initialState describes the location of the database on disk after the run
+	// completed. It can be used to seed a new run of the metamorphic test in a
+	// later Pebble version.
+	initialState initialState
+}
+
+type initialState struct {
+	desc string
+	path string
+}
+
+func (s initialState) String() string {
+	if s.desc == "" {
+		return "<empty>"
+	}
+	return s.desc
+}
+
+type subrunResults []subrunResult
+
+func (r subrunResults) historyPaths() []string {
+	paths := make([]string, len(r))
+	for i := range r {
+		paths[i] = r[i].historyPath
+	}
+	return paths
+}
+
+func (r subrunResults) initialStates() []initialState {
+	states := make([]initialState, len(r))
+	for i := range r {
+		states[i] = r[i].initialState
+	}
+	return states
+}
+
 func runVersion(
 	ctx context.Context,
 	t *testing.T,
@@ -217,7 +252,7 @@ func runVersion(
 	vers pebbleVersion,
 	seed uint64,
 	initialStates []initialState,
-) (histories []string, nextInitialStates []initialState, err error) {
+) (results subrunResults) {
 	// mu guards histories and nextInitialStates. The subtests may be run in
 	// parallel (via t.Parallel()).
 	var mu sync.Mutex
@@ -248,7 +283,8 @@ func runVersion(
 				t.Logf("  Running test with version %s with initial state %s.",
 					vers.SHA, s)
 				if err := r.run(ctx, out); err != nil {
-					fatalf(t, fatalOnce, rootDir, "Metamorphic test failed: %s\nOutput:%s\n", err, buf.String())
+					fatalf(t, fatalOnce, []string{r.dir},
+						"Metamorphic test failed: %s\nOutput:%s\n", err, buf.String())
 				}
 
 				// dir is a directory containing the ops file and subdirectories for
@@ -269,37 +305,46 @@ func runVersion(
 				mu.Lock()
 				defer mu.Unlock()
 				for _, subrunDir := range subrunDirs {
-					// Record the subrun as an initial state for the next version.
-					nextInitialStates = append(nextInitialStates, initialState{
-						path: filepath.Join(dir, subrunDir),
-						desc: fmt.Sprintf("sha=%s-seed=%d-opts=%s(%s)", vers.SHA, seed, subrunDir, s.String()),
+					results = append(results, subrunResult{
+						runDir:      dir,
+						historyPath: filepath.Join(dir, subrunDir, "history"),
+						initialState: initialState{
+							path: filepath.Join(dir, subrunDir),
+							desc: fmt.Sprintf("sha=%s-seed=%d-opts=%s(%s)", vers.SHA, seed, subrunDir, s.String()),
+						},
 					})
-					histories = append(histories, filepath.Join(dir, subrunDir, "history"))
 				}
 			})
 		}
 	})
-	return histories, nextInitialStates, err
+	return results
 }
 
-func fatalf(t testing.TB, fatalOnce *sync.Once, dir string, msg string, args ...interface{}) {
+func fatalf(t testing.TB, fatalOnce *sync.Once, dirs []string, msg string, args ...interface{}) {
 	fatalOnce.Do(func() {
 		if artifactsDir == "" {
 			var err error
 			artifactsDir, err = os.Getwd()
 			require.NoError(t, err)
 		}
-		// When run with test parallelism, other subtests may still be running
-		// within subdirectories of `dir`. We copy instead of rename so that those
-		// substests don't also fail when we remove their files out from under them.
-		// Those additional failures would confuse the test output.
-		dst := filepath.Join(artifactsDir, filepath.Base(dir))
-		t.Logf("Copying test dir %q to %q.", dir, dst)
-		_, err := vfs.Clone(vfs.Default, vfs.Default, dir, dst, vfs.CloneTryLink)
-		if err != nil {
-			t.Error(err)
+		// De-duplicate the directories.
+		slices.Sort(dirs)
+		dirs = slices.Compact(dirs)
+
+		for _, dir := range dirs {
+			// When run with test parallelism, other subtests may still be
+			// running within subdirectories of `dir`. We copy instead of rename
+			// so that those substests don't also fail when we remove their
+			// files out from under them.  Those additional failures would
+			// confuse the test output.
+			dst := filepath.Join(artifactsDir, filepath.Base(dir))
+			t.Logf("Copying test dir %q to %q.", dir, dst)
+			_, err := vfs.Clone(vfs.Default, vfs.Default, dir, dst, vfs.CloneTryLink)
+			if err != nil {
+				t.Error(err)
+			}
+			t.Fatalf(msg, args...)
 		}
-		t.Fatalf(msg, args...)
 	})
 }
 
diff --git a/internal/metamorphic/meta_test.go b/internal/metamorphic/meta_test.go
index d459b1a6..5d3415aa 100644
--- a/internal/metamorphic/meta_test.go
+++ b/internal/metamorphic/meta_test.go
@@ -51,19 +51,28 @@ var runOnceFlags, runFlags = metaflags.InitAllFlags()
 // be customized via the --inner-binary flag (used for code coverage
 // instrumentation).
 func TestMeta(t *testing.T) {
-	runTestMeta(t, false /* multiInstance */)
+	runTestMeta(t)
 }
 
 func TestMetaTwoInstance(t *testing.T) {
-	runTestMeta(t, true /* multiInstance */)
+	runTestMeta(t, metamorphic.MultiInstance(2))
 }
 
-func runTestMeta(t *testing.T, multiInstance bool) {
+func TestMetaCockroachKVs(t *testing.T) {
+	runTestMeta(t, metamorphic.CockroachKeyFormat)
+}
+
+type option interface {
+	metamorphic.RunOnceOption
+	metamorphic.RunOption
+}
+
+func runTestMeta(t *testing.T, addtlOptions ...option) {
 	switch {
 	case runOnceFlags.Compare != "":
 		onceOpts := runOnceFlags.MakeRunOnceOptions()
-		if multiInstance {
-			onceOpts = append(onceOpts, metamorphic.MultiInstance(2))
+		for _, opt := range addtlOptions {
+			onceOpts = append(onceOpts, opt)
 		}
 		testRootDir, runSubdirs := runOnceFlags.ParseCompare()
 		if runOnceFlags.TryToReduce {
@@ -78,8 +87,8 @@ func runTestMeta(t *testing.T, multiInstance bool) {
 		// runOptions() below) or the user specified it manually in order to re-run
 		// a test.
 		onceOpts := runOnceFlags.MakeRunOnceOptions()
-		if multiInstance {
-			onceOpts = append(onceOpts, metamorphic.MultiInstance(2))
+		for _, opt := range addtlOptions {
+			onceOpts = append(onceOpts, opt)
 		}
 		if runOnceFlags.TryToReduce {
 			tryToReduce(t, runOnceFlags.Dir, runOnceFlags.RunDir, runOnceFlags.ReduceAttempts)
@@ -90,8 +99,8 @@ func runTestMeta(t *testing.T, multiInstance bool) {
 
 	default:
 		opts := runFlags.MakeRunOptions()
-		if multiInstance {
-			opts = append(opts, metamorphic.MultiInstance(2))
+		for _, opt := range addtlOptions {
+			opts = append(opts, opt)
 		}
 		metamorphic.RunAndCompare(t, runFlags.Dir, opts...)
 	}
diff --git a/internal/testkeys/testkeys.go b/internal/testkeys/testkeys.go
index 35eb6144..4ddab98b 100644
--- a/internal/testkeys/testkeys.go
+++ b/internal/testkeys/testkeys.go
@@ -22,6 +22,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 )
 
@@ -106,7 +107,20 @@ var Comparer = &base.Comparer{
 		return append(append(dst, a...), 0x00)
 	},
 	Split: split,
-	Name:  "pebble.internal.testkeys",
+	ValidateKey: func(k []byte) error {
+		// Ensure that if the key has a suffix, it's a valid integer
+		// (potentially modulo a faux synthetic bit suffix).
+		k = bytes.TrimSuffix(k, ignoreTimestampSuffix)
+		i := split(k)
+		if i == len(k) {
+			return nil
+		}
+		if _, err := parseUintBytes(k[i+1:], 10, 64); err != nil {
+			return errors.Wrapf(err, "invalid key %q", k)
+		}
+		return nil
+	},
+	Name: "pebble.internal.testkeys",
 }
 
 // The comparator is similar to the one in Cockroach; when the prefixes are
diff --git a/iterator_test.go b/iterator_test.go
index ec1addbb..40144951 100644
--- a/iterator_test.go
+++ b/iterator_test.go
@@ -2715,11 +2715,12 @@ func buildFragmentedRangeKey(b testing.TB, seed uint64) (d *DB, keys [][]byte) {
 //
 // See cockroachdb/cockroach#89327.
 func BenchmarkSeekPrefixTombstones(b *testing.B) {
-	o := (&Options{
+	o := &Options{
 		FS:                 vfs.NewMem(),
 		Comparer:           testkeys.Comparer,
 		FormatMajorVersion: FormatNewest,
-	}).EnsureDefaults()
+	}
+	o.EnsureDefaults()
 	d, err := Open("", o)
 	require.NoError(b, err)
 	defer func() { require.NoError(b, d.Close()) }()
@@ -2777,11 +2778,13 @@ func BenchmarkPointDeletedSwath(b *testing.B) {
 	ks := testkeys.Alpha(maxKeyLen)
 
 	opts := func() *Options {
-		return (&Options{
+		o := &Options{
 			FS:                 vfs.NewMem(),
 			Comparer:           testkeys.Comparer,
 			FormatMajorVersion: FormatNewest,
-		}).EnsureDefaults()
+		}
+		o.EnsureDefaults()
+		return o
 	}
 	type iteratorOp struct {
 		name string
@@ -2963,13 +2966,14 @@ func runBenchmarkQueueWorkload(b *testing.B, deleteRatio float32, initOps int, v
 		queues[i] = &Queue{}
 	}
 
-	o := (&Options{
+	o := &Options{
 		Cache:              cache.New(0), // disable cache
 		DisableWAL:         true,
 		FS:                 vfs.NewMem(),
 		Comparer:           testkeys.Comparer,
 		FormatMajorVersion: FormatNewest,
-	}).EnsureDefaults()
+	}
+	o.EnsureDefaults()
 
 	d, err := Open("", o)
 	require.NoError(b, err)
diff --git a/level_checker_test.go b/level_checker_test.go
index 6961c7a9..d632cff7 100644
--- a/level_checker_test.go
+++ b/level_checker_test.go
@@ -102,7 +102,7 @@ func TestCheckLevelsCornerCases(t *testing.T) {
 		}
 	}()
 
-	var fileNum FileNum
+	var fileNum base.FileNum
 	newIters :=
 		func(_ context.Context, file *manifest.FileMetadata, _ *IterOptions, iio internalIterOpts, _ iterKinds) (iterSet, error) {
 			r := readers[file.FileNum]
diff --git a/level_iter_test.go b/level_iter_test.go
index 0d3f1d76..7a78f7f5 100644
--- a/level_iter_test.go
+++ b/level_iter_test.go
@@ -61,7 +61,7 @@ func TestLevelIter(t *testing.T) {
 				iterKVs = append(iterKVs, kvs)
 
 				meta := (&fileMetadata{
-					FileNum: FileNum(len(metas)),
+					FileNum: base.FileNum(len(metas)),
 				}).ExtendPointKeyBounds(
 					DefaultComparer.Compare,
 					kvs[0].K,
@@ -213,7 +213,7 @@ func (lt *levelIterTest) runClear() string {
 }
 
 func (lt *levelIterTest) runBuild(d *datadriven.TestData) string {
-	fileNum := FileNum(len(lt.readers))
+	fileNum := base.FileNum(len(lt.readers))
 	name := fmt.Sprint(fileNum)
 	f0, err := lt.mem.Create(name, vfs.WriteCategoryUnspecified)
 	if err != nil {
@@ -542,8 +542,10 @@ func buildLevelIterTables(
 	}
 	c := NewCache(128 << 20 /* 128MB */)
 	defer c.Unref()
+	ch := c.NewHandle()
+	defer ch.Close()
 	opts := sstable.ReaderOptions{Comparer: DefaultComparer}
-	opts.CacheOpts = sstableinternal.CacheOptions{Cache: c}
+	opts.CacheOpts = sstableinternal.CacheOptions{CacheHandle: ch}
 	readers := make([]*sstable.Reader, len(files))
 	for i := range files {
 		f, err := mem.Open(fmt.Sprintf("bench%d", i))
@@ -572,7 +574,7 @@ func buildLevelIterTables(
 		require.NoError(b, err)
 		smallest := iter.First()
 		meta[i] = &fileMetadata{}
-		meta[i].FileNum = FileNum(i)
+		meta[i].FileNum = base.FileNum(i)
 		largest := iter.Last()
 		meta[i].ExtendPointKeyBounds(opts.Comparer.Compare, smallest.K.Clone(), largest.K.Clone())
 		meta[i].InitPhysicalBacking()
diff --git a/merging_iter_test.go b/merging_iter_test.go
index afd766af..8cbdf0e1 100644
--- a/merging_iter_test.go
+++ b/merging_iter_test.go
@@ -145,7 +145,7 @@ func TestMergingIterDataDriven(t *testing.T) {
 	memFS := vfs.NewMem()
 	cmp := DefaultComparer.Compare
 	fmtKey := DefaultComparer.FormatKey
-	opts := (*Options)(nil).EnsureDefaults()
+	opts := DefaultOptions()
 	var v *version
 	var buf bytes.Buffer
 
@@ -370,9 +370,11 @@ func buildMergingIterTables(
 	}
 	c := NewCache(128 << 20 /* 128MB */)
 	defer c.Unref()
+	ch := c.NewHandle()
+	defer ch.Close()
 
 	var opts sstable.ReaderOptions
-	opts.CacheOpts = sstableinternal.CacheOptions{Cache: c}
+	opts.CacheOpts = sstableinternal.CacheOptions{CacheHandle: ch}
 
 	readers := make([]*sstable.Reader, len(files))
 	for i := range files {
@@ -613,9 +615,11 @@ func buildLevelsForMergingIterSeqSeek(
 
 	c := NewCache(128 << 20 /* 128MB */)
 	defer c.Unref()
+	ch := c.NewHandle()
+	defer ch.Close()
 
 	opts := sstable.ReaderOptions{Comparer: DefaultComparer}
-	opts.CacheOpts = sstableinternal.CacheOptions{Cache: c}
+	opts.CacheOpts = sstableinternal.CacheOptions{CacheHandle: ch}
 
 	if writeBloomFilters {
 		opts.Filters = make(map[string]FilterPolicy)
@@ -651,7 +655,7 @@ func buildLevelsForMergingIterSeqSeek(
 			// The same FileNum is being reused across different levels, which
 			// is harmless for the benchmark since each level has its own iterator
 			// creation func.
-			meta[j].FileNum = FileNum(j)
+			meta[j].FileNum = base.FileNum(j)
 			largest := iter.Last()
 			meta[j].ExtendPointKeyBounds(opts.Comparer.Compare, smallest.K.Clone(), largest.K.Clone())
 			meta[j].InitPhysicalBacking()
diff --git a/metamorphic/generator_test.go b/metamorphic/generator_test.go
index 12efc8e1..88114f56 100644
--- a/metamorphic/generator_test.go
+++ b/metamorphic/generator_test.go
@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/cockroachdb/pebble/cockroachkvs"
 	"github.com/cockroachdb/pebble/internal/randvar"
 	"github.com/pmezard/go-difflib/difflib"
 	"github.com/stretchr/testify/require"
@@ -194,3 +195,88 @@ func TestGenerateDisjointKeyRanges(t *testing.T) {
 		}
 	}
 }
+
+func TestCockroachSuffixKeyspace(t *testing.T) {
+	testCasesByMaxLogical := [][]string{
+		0: {
+			"",
+			"0.000000001,0",
+			"0.000000002,0",
+			"0.000000003,0",
+			"0.000000004,0",
+			"0.000000005,0",
+			"0.000000006,0",
+			"0.000000007,0",
+		},
+		1: {
+			"",
+			"0,1",
+			"0.000000001,0",
+			"0.000000001,1",
+			"0.000000002,0",
+			"0.000000002,1",
+			"0.000000003,0",
+			"0.000000003,1",
+			"0.000000004,0",
+			"0.000000004,1",
+		},
+		2: {
+			"",
+			"0,1",
+			"0,2",
+			"0.000000001,0",
+			"0.000000001,1",
+			"0.000000001,2",
+			"0.000000002,0",
+			"0.000000002,1",
+			"0.000000002,2",
+			"0.000000003,0",
+			"0.000000003,1",
+			"0.000000003,2",
+			"0.000000004,0",
+			"0.000000004,1",
+			"0.000000004,2",
+		},
+		3: {
+			"",
+			"0,1",
+			"0,2",
+			"0,3",
+			"0.000000001,0",
+			"0.000000001,1",
+			"0.000000001,2",
+			"0.000000001,3",
+			"0.000000002,0",
+			"0.000000002,1",
+			"0.000000002,2",
+			"0.000000002,3",
+			"0.000000003,0",
+			"0.000000003,1",
+			"0.000000003,2",
+			"0.000000003,3",
+			"0.000000004,0",
+			"0.000000004,1",
+			"0.000000004,2",
+			"0.000000004,3",
+		},
+	}
+	for maxLogical, testCases := range testCasesByMaxLogical {
+		t.Run(fmt.Sprintf("maxLogical=%d", maxLogical), func(t *testing.T) {
+			ks := cockroachSuffixKeyspace{maxLogical: int64(maxLogical)}
+
+			for si, expect := range testCases {
+				suffix := ks.ToMaterializedSuffix(suffixIndex(si))
+				var prettyPrinted string
+				if len(suffix) > 0 {
+					prettyPrinted = fmt.Sprint(cockroachkvs.FormatKeySuffix(suffix))
+				}
+				if expect != prettyPrinted {
+					t.Errorf("ToMaterializedSuffix(%d) = %q, want %q", si, prettyPrinted, expect)
+				}
+				if got := ks.ToSuffixIndex(suffix); got != suffixIndex(si) {
+					t.Errorf("ToSuffixIndex(%q) = %d, want %d", prettyPrinted, got, si)
+				}
+			}
+		})
+	}
+}
diff --git a/metamorphic/options_test.go b/metamorphic/options_test.go
index 827ca323..9b016cd4 100644
--- a/metamorphic/options_test.go
+++ b/metamorphic/options_test.go
@@ -93,21 +93,12 @@ func TestOptionsRoundtrip(t *testing.T) {
 		"Experimental.MultiLevelCompactionHeuristic.AddPropensity",
 	}
 
-	// Ensure that we unref any caches created, so invariants builds don't
-	// complain about the leaked ref counts.
-	maybeUnref := func(o *TestOptions) {
-		if o.Opts.Cache != nil {
-			o.Opts.Cache.Unref()
-		}
-	}
-
 	checkOptions := func(t *testing.T, o *TestOptions) {
 		s := optionsToString(o)
 		t.Logf("Serialized options:\n%s\n", s)
 
 		parsed := defaultTestOptions(TestkeysKeyFormat)
 		require.NoError(t, parseOptions(parsed, s, nil))
-		maybeUnref(parsed)
 		got := optionsToString(parsed)
 		require.Equal(t, s, got)
 		t.Logf("Re-serialized options:\n%s\n", got)
@@ -143,13 +134,12 @@ func TestOptionsRoundtrip(t *testing.T) {
 				cleaned = append(cleaned, d)
 			}
 		}
-		require.Equal(t, diff[:0], cleaned)
+		require.Equal(t, diff[:0], cleaned, "before:\n%#v\nafter:\n%#v\n", o.Opts, parsed.Opts)
 	}
 
 	standard := standardOptions(TestkeysKeyFormat)
 	for i := range standard {
 		t.Run(fmt.Sprintf("standard-%03d", i), func(t *testing.T) {
-			defer maybeUnref(standard[i])
 			checkOptions(t, standard[i])
 		})
 	}
@@ -157,7 +147,6 @@ func TestOptionsRoundtrip(t *testing.T) {
 	for i := 0; i < 100; i++ {
 		t.Run(fmt.Sprintf("random-%03d", i), func(t *testing.T) {
 			o := RandomOptions(rng, TestkeysKeyFormat, nil)
-			defer maybeUnref(o)
 			checkOptions(t, o)
 		})
 	}
@@ -234,7 +223,6 @@ func TestCustomOptionParser(t *testing.T) {
   foo=bar
 `, customOptionParsers))
 	require.NoError(t, parseOptions(o2, optionsToString(o1), customOptionParsers))
-	defer o2.Opts.Cache.Unref()
 
 	for _, o := range []*TestOptions{o1, o2} {
 		require.Equal(t, 1, len(o.CustomOpts))
diff --git a/metamorphic/test.go b/metamorphic/test.go
index 1ccf4bb4..8091b817 100644
--- a/metamorphic/test.go
+++ b/metamorphic/test.go
@@ -93,7 +93,8 @@ func (t *Test) init(
 		t.writeOpts = pebble.NoSync
 	}
 	testOpts.Opts.WithFSDefaults()
-	t.opts = testOpts.Opts.EnsureDefaults()
+	t.opts = testOpts.Opts.Clone()
+	t.opts.EnsureDefaults()
 	t.opts.Logger = h
 	lel := pebble.MakeLoggingEventListener(t.opts.Logger)
 	t.opts.EventListener = &lel
@@ -115,7 +116,9 @@ func (t *Test) init(
 
 	t.opsWaitOn, t.opsDone = computeSynchronizationPoints(t.ops)
 
-	defer t.opts.Cache.Unref()
+	if t.opts.Cache != nil {
+		defer t.opts.Cache.Unref()
+	}
 
 	// If an error occurs and we were using an in-memory FS, attempt to clone to
 	// on-disk in order to allow post-mortem debugging. Note that always using
@@ -285,7 +288,6 @@ func (t *Test) restartDB(dbID objID) error {
 	if len(t.dbs) > 1 {
 		return nil
 	}
-	t.opts.Cache.Ref()
 	fs := vfs.Root(t.opts.FS).(*vfs.MemFS)
 	crashFS := fs.CrashClone(vfs.CrashCloneCfg{UnsyncedDataPercent: 0})
 	if err := db.Close(); err != nil {
@@ -326,7 +328,6 @@ func (t *Test) restartDB(dbID objID) error {
 		}
 		return err
 	})
-	t.opts.Cache.Unref()
 	return err
 }
 
diff --git a/metamorphic/testkeys.go b/metamorphic/testkeys.go
index 0e60d94c..41506fda 100644
--- a/metamorphic/testkeys.go
+++ b/metamorphic/testkeys.go
@@ -10,6 +10,7 @@ import (
 	"math/rand/v2"
 	"slices"
 
+	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble"
 	"github.com/cockroachdb/pebble/internal/testkeys"
 	"github.com/cockroachdb/pebble/sstable"
@@ -39,15 +40,26 @@ var TestkeysKeyFormat = KeyFormat{
 		return sstable.NewTestKeysMaskingFilter()
 	},
 	NewSuffixBlockPropertyFilter: func(filterMin, filterMax []byte) sstable.BlockPropertyFilter {
-		low, err := testkeys.ParseSuffix(filterMin)
-		if err != nil {
-			panic(err)
+		var low, high int64
+		var err error
+		if filterMin != nil {
+			low, err = testkeys.ParseSuffix(filterMin)
+			if err != nil {
+				panic(err)
+			}
 		}
-		high, err := testkeys.ParseSuffix(filterMax)
-		if err != nil {
-			panic(err)
+		if filterMax != nil {
+			high, err = testkeys.ParseSuffix(filterMax)
+			if err != nil {
+				panic(err)
+			}
+		}
+		// The suffixes were encoded in descending order, so low should be the
+		// max timestamp and high should be the min timestamp.
+		if low <= high {
+			panic(errors.AssertionFailedf("low <= high: %d <= %d", low, high))
 		}
-		return sstable.NewTestKeysBlockPropertyFilter(uint64(low), uint64(high))
+		return sstable.NewTestKeysBlockPropertyFilter(uint64(high), uint64(low))
 	},
 }
 
@@ -144,15 +156,28 @@ func (kg *testkeyKeyGenerator) IncMaxSuffix() []byte {
 	return testkeys.Suffix(int64(kg.cfg.writeSuffixDist.Max()))
 }
 
+// SuffixRange generates a new uniformly random range of suffixes (low, high]
+// such that high is guaranteed to be strictly greater (as defined by
+// ComparePointSuffixes) than low.
+//
+// The high suffix may be nil, in which case the suffix range represents all
+// suffixes ≥ low.
 func (kg *testkeyKeyGenerator) SuffixRange() (low, high []byte) {
 	a := kg.uniformSuffixInt()
 	b := kg.uniformSuffixInt()
-	if a > b {
+	// NB: Suffixes are sorted in descending order, so we need to generate the
+	// timestamps such that a > b. This ensures that the returned suffixes sort
+	// such that low < high.
+	if a < b {
 		a, b = b, a
 	} else if a == b {
-		b++
+		a++
+	}
+	low = testkeys.Suffix(a) // NB: a > 0
+	if b > 0 {
+		high = testkeys.Suffix(b)
 	}
-	return testkeys.Suffix(a), testkeys.Suffix(b)
+	return low, high
 }
 
 // UniformSuffix returns a suffix in the same range as SkewedSuffix but with a
diff --git a/objstorage/objstorageprovider/provider_test.go b/objstorage/objstorageprovider/provider_test.go
index 0c8f9f02..1c371b46 100644
--- a/objstorage/objstorageprovider/provider_test.go
+++ b/objstorage/objstorageprovider/provider_test.go
@@ -110,6 +110,11 @@ func TestProvider(t *testing.T) {
 				opts := objstorage.CreateOptions{
 					SharedCleanupMethod: objstorage.SharedRefTracking,
 				}
+				ft := base.FileTypeTable
+				if len(d.CmdArgs) > 0 && d.CmdArgs[0].Key == "file-type" {
+					ft = base.FileTypeFromName(d.CmdArgs[0].FirstVal(t))
+					d.CmdArgs = d.CmdArgs[1:]
+				}
 				if len(d.CmdArgs) == 5 && d.CmdArgs[4].Key == "no-ref-tracking" {
 					d.CmdArgs = d.CmdArgs[:4]
 					opts.SharedCleanupMethod = objstorage.SharedNoCleanup
@@ -117,7 +122,7 @@ func TestProvider(t *testing.T) {
 				var fileNum base.DiskFileNum
 				var typ string
 				var salt, size int
-				scanArgs("<file-num> <local|shared> <salt> <size> [no-ref-tracking]", &fileNum, &typ, &salt, &size)
+				scanArgs("[file-type=sstable|blob] <file-num> <local|shared> <salt> <size> [no-ref-tracking]", &fileNum, &typ, &salt, &size)
 				switch typ {
 				case "local":
 				case "shared":
@@ -125,7 +130,7 @@ func TestProvider(t *testing.T) {
 				default:
 					d.Fatalf(t, "'%s' should be 'local' or 'shared'", typ)
 				}
-				w, _, err := curProvider.Create(ctx, base.FileTypeTable, fileNum, opts)
+				w, _, err := curProvider.Create(ctx, ft, fileNum, opts)
 				if err != nil {
 					return err.Error()
 				}
@@ -141,6 +146,11 @@ func TestProvider(t *testing.T) {
 				opts := objstorage.CreateOptions{
 					SharedCleanupMethod: objstorage.SharedRefTracking,
 				}
+				ft := base.FileTypeTable
+				if len(d.CmdArgs) > 0 && d.CmdArgs[0].Key == "file-type" {
+					ft = base.FileTypeFromName(d.CmdArgs[0].FirstVal(t))
+					d.CmdArgs = d.CmdArgs[1:]
+				}
 				if len(d.CmdArgs) == 5 && d.CmdArgs[4].Key == "no-ref-tracking" {
 					d.CmdArgs = d.CmdArgs[:4]
 					opts.SharedCleanupMethod = objstorage.SharedNoCleanup
@@ -148,7 +158,7 @@ func TestProvider(t *testing.T) {
 				var fileNum base.DiskFileNum
 				var typ string
 				var salt, size int
-				scanArgs("<file-num> <local|shared> <salt> <size> [no-ref-tracking]", &fileNum, &typ, &salt, &size)
+				scanArgs("[file-type=sstable|blob] <file-num> <local|shared> <salt> <size> [no-ref-tracking]", &fileNum, &typ, &salt, &size)
 				switch typ {
 				case "local":
 				case "shared":
@@ -168,9 +178,7 @@ func TestProvider(t *testing.T) {
 				require.NoError(t, err)
 				require.NoError(t, f.Close())
 
-				_, err = curProvider.LinkOrCopyFromLocal(
-					ctx, fs, tmpFilename, base.FileTypeTable, fileNum, opts,
-				)
+				_, err = curProvider.LinkOrCopyFromLocal(ctx, fs, tmpFilename, ft, fileNum, opts)
 				require.NoError(t, err)
 				return log.String()
 
@@ -195,10 +203,15 @@ func TestProvider(t *testing.T) {
 					}
 				}
 
+				ft := base.FileTypeTable
+				if len(d.CmdArgs) > 0 && d.CmdArgs[0].Key == "file-type" {
+					ft = base.FileTypeFromName(d.CmdArgs[0].FirstVal(t))
+					d.CmdArgs = d.CmdArgs[1:]
+				}
 				d.CmdArgs = d.CmdArgs[:1]
 				var fileNum base.DiskFileNum
-				scanArgs("<file-num> [for-compaction] [readahead|speculative-overhead=off|sys-readahead|fadvise-sequential]", &fileNum)
-				r, err := curProvider.OpenForReading(ctx, base.FileTypeTable, fileNum, objstorage.OpenOptions{})
+				scanArgs("[file-type=sstable|blob] <file-num> [for-compaction] [readahead|speculative-overhead=off|sys-readahead|fadvise-sequential]", &fileNum)
+				r, err := curProvider.OpenForReading(ctx, ft, fileNum, objstorage.OpenOptions{})
 				if err != nil {
 					return err.Error()
 				}
@@ -231,9 +244,14 @@ func TestProvider(t *testing.T) {
 				return log.String()
 
 			case "remove":
+				ft := base.FileTypeTable
+				if len(d.CmdArgs) > 0 && d.CmdArgs[0].Key == "file-type" {
+					ft = base.FileTypeFromName(d.CmdArgs[0].FirstVal(t))
+					d.CmdArgs = d.CmdArgs[1:]
+				}
 				var fileNum base.DiskFileNum
-				scanArgs("<file-num>", &fileNum)
-				if err := curProvider.Remove(base.FileTypeTable, fileNum); err != nil {
+				scanArgs("[file-type=sstable|blob] <file-num>", &fileNum)
+				if err := curProvider.Remove(ft, fileNum); err != nil {
 					return err.Error()
 				}
 				return log.String()
diff --git a/objstorage/objstorageprovider/testdata/provider/local b/objstorage/objstorageprovider/testdata/provider/local
index c2bba671..3d7f1899 100644
--- a/objstorage/objstorageprovider/testdata/provider/local
+++ b/objstorage/objstorageprovider/testdata/provider/local
@@ -62,7 +62,7 @@ list
 
 read 1
 ----
-file 000001 (type 2) unknown to the objstorage provider: file does not exist
+file 000001 (type sstable) unknown to the objstorage provider: file does not exist
 
 link-or-copy 3 local 3 100
 ----
@@ -94,6 +94,38 @@ size: 1234
 0 1234: ok (salt 4)
 <local fs> close: p0/000004.sst
 
+create file-type=blob 000005 local 1 4096
+----
+<local fs> create: p0/000005.blob
+<local fs> sync-data: p0/000005.blob
+<local fs> close: p0/000005.blob
+
+read file-type=blob 000005
+0 1024
+2048 1024
+----
+<local fs> open: p0/000005.blob (options: *vfs.randomReadsOption)
+size: 4096
+<local fs> read-at(0, 1024): p0/000005.blob
+0 1024: ok (salt 1)
+<local fs> read-at(2048, 1024): p0/000005.blob
+2048 1024: ok (salt 1)
+<local fs> close: p0/000005.blob
+
+link-or-copy file-type=blob 000006 shared 6 1234
+----
+<local fs> create: temp-file-3
+<local fs> close: temp-file-3
+<local fs> link: temp-file-3 -> p0/000006.blob
+
+list
+----
+000002 -> p0/000002.sst
+000003 -> p0/000003.sst
+000004 -> p0/000004.sst
+000005 -> p0/000005.blob
+000006 -> p0/000006.blob
+
 close
 ----
 <local fs> sync: p0
diff --git a/obsolete_files_test.go b/obsolete_files_test.go
index 1e7ea2f1..11095209 100644
--- a/obsolete_files_test.go
+++ b/obsolete_files_test.go
@@ -92,11 +92,12 @@ func TestCleaner(t *testing.T) {
 				return "open <dir> [archive] [readonly]"
 			}
 			dir := td.CmdArgs[0].String()
-			opts := (&Options{
+			opts := &Options{
 				FS:     fs,
 				WALDir: dir + "_wal",
 				Logger: testLogger{t},
-			}).WithFSDefaults()
+			}
+			opts.WithFSDefaults()
 
 			for i := 1; i < len(td.CmdArgs); i++ {
 				switch td.CmdArgs[i].String() {
diff --git a/open_test.go b/open_test.go
index 0e9dddec..958d4945 100644
--- a/open_test.go
+++ b/open_test.go
@@ -50,7 +50,7 @@ import (
 
 func TestOpenSharedFileCache(t *testing.T) {
 	c := cache.New(cacheDefaultSize)
-	tc := NewFileCache(c, 16, 100)
+	tc := NewFileCache(16, 100)
 	defer tc.Unref()
 	defer c.Unref()
 
@@ -443,7 +443,7 @@ func testOpenCloseOpenClose(t *testing.T, fs vfs.FS, root string) {
 					}
 					var optionsCount int
 					for _, s := range got {
-						if t, _, ok := base.ParseFilename(opts.FS, s); ok && t == fileTypeOptions {
+						if t, _, ok := base.ParseFilename(opts.FS, s); ok && t == base.FileTypeOptions {
 							optionsCount++
 						}
 					}
@@ -994,33 +994,18 @@ func TestCrashOpenCrashAfterWALCreation(t *testing.T) {
 	// file is created and after a subsequent directory sync occurs. This
 	// simulates a crash after the new log file is created and synced.
 	crashFS = nil
-	var crashFSAtomic atomic.Pointer[vfs.MemFS]
+	instrumentedFS := &crashAfterLogCreationFS{MemFS: fs}
 	{
-		var walCreated, dirSynced atomic.Bool
 		d, err := Open("", &Options{
-			FS: errorfs.Wrap(fs, errorfs.InjectorFunc(func(op errorfs.Op) error {
-				if dirSynced.Load() {
-					crashFSAtomic.CompareAndSwap(nil, fs.CrashClone(vfs.CrashCloneCfg{UnsyncedDataPercent: 0}))
-				}
-				if op.Kind == errorfs.OpCreate && filepath.Ext(op.Path) == ".log" {
-					walCreated.Store(true)
-				}
-				// Record when there's a sync of the data directory after the
-				// WAL was created. The data directory will have an empty
-				// path because that's what we passed into Open.
-				if op.Kind == errorfs.OpFileSync && op.Path == "" && walCreated.Load() {
-					dirSynced.Store(true)
-				}
-				return nil
-			})),
+			FS:     instrumentedFS,
 			Logger: testLogger{t},
 		})
 		require.NoError(t, err)
 		require.NoError(t, d.Close())
 	}
 
-	require.NotNil(t, crashFSAtomic.Load())
-	fs = crashFSAtomic.Load()
+	fs = instrumentedFS.saved
+	require.NotNil(t, fs)
 
 	newLogs := getLogs()
 	if n := len(newLogs); n > 2 || n < 1 {
@@ -1033,7 +1018,10 @@ func TestCrashOpenCrashAfterWALCreation(t *testing.T) {
 		require.NoError(t, err)
 		curLogNum, err := strconv.Atoi(strings.Split(newLogs[0], ".")[0])
 		require.NoError(t, err)
-		require.Greater(t, curLogNum, origLogNum)
+		if curLogNum == origLogNum {
+			t.Logf("logs: %v newLogs: %v", logs, newLogs)
+		}
+		require.Greater(t, curLogNum, origLogNum, "%v %v", logs, newLogs)
 	}
 
 	// Finally, open the database with syncs enabled.
@@ -1042,6 +1030,59 @@ func TestCrashOpenCrashAfterWALCreation(t *testing.T) {
 	require.NoError(t, d.Close())
 }
 
+// crashAfterLogCreationFS wraps a MemFS and extracts a crash clone after a WAL
+// is created and the directory is subsequently synced.
+// TODO(radu): replace this with errorfs predicates; see
+// https://github.com/cockroachdb/pebble/pull/4308#pullrequestreview-2584817912
+type crashAfterLogCreationFS struct {
+	*vfs.MemFS
+	// state is 0 before the first log is created, 1 after the first log is
+	// created, 2 after the dir is synced.
+	state atomic.Int32
+	saved *vfs.MemFS
+}
+
+// crashAfterLogCreationDir wraps the File for the store directory to instrument
+// its Sync method.
+type crashAfterLogCreationDir struct {
+	vfs.File
+
+	fs *crashAfterLogCreationFS
+}
+
+func (fs *crashAfterLogCreationFS) Create(
+	name string, category vfs.DiskWriteCategory,
+) (vfs.File, error) {
+	f, err := fs.MemFS.Create(name, category)
+	if err == nil && filepath.Ext(name) == ".log" {
+		fs.state.CompareAndSwap(0, 1)
+	}
+	return f, err
+}
+
+func (fs *crashAfterLogCreationFS) OpenDir(fullname string) (vfs.File, error) {
+	f, err := fs.MemFS.OpenDir(fullname)
+	// The store directory will have an empty name (this is what we pass to Open).
+	if err != nil || fullname != "" {
+		return f, err
+	}
+	// Wrap the store directory File.
+	return &crashAfterLogCreationDir{File: f, fs: fs}, nil
+}
+
+func (d *crashAfterLogCreationDir) Sync() error {
+	if err := d.File.Sync(); err != nil {
+		return err
+	}
+	// Record the first time there's a sync of the data directory after the WAL
+	// was created.
+	if d.fs.state.CompareAndSwap(1, 2) {
+		// The atomic 1->2 transition can only happen once, so there is no race.
+		d.fs.saved = d.fs.MemFS.CrashClone(vfs.CrashCloneCfg{UnsyncedDataPercent: 0})
+	}
+	return nil
+}
+
 // TestOpenWALReplayReadOnlySeqNums tests opening a database:
 //   - in read-only mode
 //   - with multiple unflushed log files that must replayed
@@ -1192,7 +1233,7 @@ func TestGetVersion(t *testing.T) {
 			continue
 		}
 		switch ft {
-		case fileTypeOptions:
+		case base.FileTypeOptions:
 			if fn > highestOptionsNum {
 				highestOptionsNum = fn
 			}
@@ -1417,7 +1458,7 @@ func TestCheckConsistency(t *testing.T) {
 				}
 
 				v := manifest.NewVersion(base.DefaultComparer, 0, filesByLevel)
-				err := checkConsistency(v, dir, provider)
+				err := checkConsistency(v, provider)
 				if err != nil {
 					if redactErr {
 						redacted := redact.Sprint(err).Redact()
@@ -1474,7 +1515,8 @@ func TestOpenRatchetsNextFileNum(t *testing.T) {
 	// Create a shared file with the newest file num and then close the db.
 	d.mu.Lock()
 	nextFileNum := d.mu.versions.getNextDiskFileNum()
-	w, _, err := d.objProvider.Create(context.TODO(), fileTypeTable, nextFileNum, objstorage.CreateOptions{PreferSharedStorage: true})
+	w, _, err := d.objProvider.Create(context.TODO(), base.FileTypeTable, nextFileNum,
+		objstorage.CreateOptions{PreferSharedStorage: true})
 	require.NoError(t, err)
 	require.NoError(t, w.Write([]byte("foobar")))
 	require.NoError(t, w.Finish())
diff --git a/options_test.go b/options_test.go
index 3eaf4380..16b5936c 100644
--- a/options_test.go
+++ b/options_test.go
@@ -5,7 +5,6 @@
 package pebble
 
 import (
-	"fmt"
 	"math/rand/v2"
 	"runtime"
 	"strings"
@@ -50,8 +49,7 @@ func testingRandomized(t testing.TB, o *Options) *Options {
 }
 
 func TestLevelOptions(t *testing.T) {
-	var opts *Options
-	opts = opts.EnsureDefaults()
+	opts := DefaultOptions()
 
 	testCases := []struct {
 		level          int
@@ -74,7 +72,7 @@ func TestLevelOptions(t *testing.T) {
 	}
 }
 
-func TestOptionsString(t *testing.T) {
+func TestDefaultOptionsString(t *testing.T) {
 	n := runtime.GOMAXPROCS(8)
 	defer runtime.GOMAXPROCS(n)
 
@@ -133,14 +131,11 @@ func TestOptionsString(t *testing.T) {
   target_file_size=2097152
 `
 
-	var opts *Options
-	opts = opts.EnsureDefaults()
-	require.Equal(t, expected, opts.String())
+	require.Equal(t, expected, DefaultOptions().String())
 }
 
 func TestOptionsCheckCompatibility(t *testing.T) {
-	var opts *Options
-	opts = opts.EnsureDefaults()
+	opts := DefaultOptions()
 	s := opts.String()
 	require.NoError(t, opts.CheckCompatibility(s))
 	require.Regexp(t, `invalid key=value syntax`, opts.CheckCompatibility("foo\n"))
@@ -183,18 +178,20 @@ func TestOptionsCheckCompatibility(t *testing.T) {
 	// Check that an OPTIONS file that configured an explicit WALDir that will
 	// no longer be used errors if it's not also present in WALRecoveryDirs.
 	require.Equal(t, ErrMissingWALRecoveryDir{Dir: "external-wal-dir"},
-		(&Options{}).EnsureDefaults().CheckCompatibility(`
+		DefaultOptions().CheckCompatibility(`
 [Options]
   wal_dir=external-wal-dir
 `))
 	// But not if it's configured as a WALRecoveryDir or current WALDir.
-	require.NoError(t,
-		(&Options{WALRecoveryDirs: []wal.Dir{{Dirname: "external-wal-dir"}}}).EnsureDefaults().CheckCompatibility(`
+	opts = &Options{WALRecoveryDirs: []wal.Dir{{Dirname: "external-wal-dir"}}}
+	opts.EnsureDefaults()
+	require.NoError(t, opts.CheckCompatibility(`
 [Options]
   wal_dir=external-wal-dir
 `))
-	require.NoError(t,
-		(&Options{WALDir: "external-wal-dir"}).EnsureDefaults().CheckCompatibility(`
+	opts = &Options{WALDir: "external-wal-dir"}
+	opts.EnsureDefaults()
+	require.NoError(t, opts.CheckCompatibility(`
 [Options]
   wal_dir=external-wal-dir
 `))
@@ -203,7 +200,7 @@ func TestOptionsCheckCompatibility(t *testing.T) {
 	// that will no longer be used errors if it's not also present in
 	// WALRecoveryDirs.
 	require.Equal(t, ErrMissingWALRecoveryDir{Dir: "failover-wal-dir"},
-		(&Options{}).EnsureDefaults().CheckCompatibility(`
+		DefaultOptions().CheckCompatibility(`
 [Options]
 
 [WAL Failover]
@@ -211,13 +208,17 @@ func TestOptionsCheckCompatibility(t *testing.T) {
 `))
 	// But not if it's configured as a WALRecoveryDir or current failover
 	// secondary dir.
-	require.NoError(t, (&Options{WALRecoveryDirs: []wal.Dir{{Dirname: "failover-wal-dir"}}}).EnsureDefaults().CheckCompatibility(`
+	opts = &Options{WALRecoveryDirs: []wal.Dir{{Dirname: "failover-wal-dir"}}}
+	opts.EnsureDefaults()
+	require.NoError(t, opts.CheckCompatibility(`
 [Options]
 
 [WAL Failover]
   secondary_dir=failover-wal-dir
 `))
-	require.NoError(t, (&Options{WALFailover: &WALFailoverOptions{Secondary: wal.Dir{Dirname: "failover-wal-dir"}}}).EnsureDefaults().CheckCompatibility(`
+	opts = &Options{WALFailover: &WALFailoverOptions{Secondary: wal.Dir{Dirname: "failover-wal-dir"}}}
+	opts.EnsureDefaults()
+	require.NoError(t, opts.CheckCompatibility(`
 [Options]
 
 [WAL Failover]
@@ -240,13 +241,8 @@ func TestOptionsParse(t *testing.T) {
 	testComparer.Name = "test-comparer"
 	testMerger := *DefaultMerger
 	testMerger.Name = "test-merger"
-	var newCacheSize int64
 
 	hooks := &ParseHooks{
-		NewCache: func(size int64) *Cache {
-			newCacheSize = size
-			return nil
-		},
 		NewCleaner: func(name string) (Cleaner, error) {
 			if name == (testCleaner{}).String() {
 				return testCleaner{}, nil
@@ -306,7 +302,6 @@ func TestOptionsParse(t *testing.T) {
 			opts.EnsureDefaults()
 			str := opts.String()
 
-			newCacheSize = 0
 			var parsedOptions Options
 			require.NoError(t, parsedOptions.Parse(str, hooks))
 			parsedStr := parsedOptions.String()
@@ -314,7 +309,6 @@ func TestOptionsParse(t *testing.T) {
 				t.Fatalf("expected\n%s\nbut found\n%s", str, parsedStr)
 			}
 			require.Nil(t, parsedOptions.Cache)
-			require.NotEqual(t, newCacheSize, 0)
 		})
 	}
 }
@@ -378,24 +372,6 @@ func TestOptionsValidate(t *testing.T) {
 	}
 }
 
-// This test isn't being done in TestOptionsValidate
-// cause it doesn't support setting pointers.
-func TestOptionsValidateCache(t *testing.T) {
-	var opts Options
-	opts.EnsureDefaults()
-	opts.Cache = NewCache(8 << 20)
-	defer opts.Cache.Unref()
-	opts.FileCache = NewFileCache(NewCache(8<<20), 10, 1)
-	defer opts.FileCache.cache.Unref()
-	defer opts.FileCache.Unref()
-
-	err := opts.Validate()
-	require.Error(t, err)
-	if fmt.Sprint(err) != "underlying cache in the FileCache and the Cache dont match" {
-		t.Errorf("Unexpected error message")
-	}
-}
-
 func TestKeyCategories(t *testing.T) {
 	kc := MakeUserKeyCategories(base.DefaultComparer.Compare, []UserKeyCategory{
 		{Name: "b", UpperBound: []byte("b")},
diff --git a/record/record_test.go b/record/record_test.go
index 885d221f..d31f31de 100644
--- a/record/record_test.go
+++ b/record/record_test.go
@@ -828,8 +828,8 @@ func TestInvalidLogNum(t *testing.T) {
 
 	{
 		r := NewReader(bytes.NewReader(buf.Bytes()), 2)
-		if _, err := r.Next(); err != io.EOF {
-			t.Fatalf("expected %s, but found %s\n", io.EOF, err)
+		if _, err := r.Next(); err != ErrInvalidChunk {
+			t.Fatalf("expected %s, but found %s\n", ErrInvalidChunk, err)
 		}
 	}
 }
diff --git a/sstable/block_property_test.go b/sstable/block_property_test.go
index 026be11d..fec783a7 100644
--- a/sstable/block_property_test.go
+++ b/sstable/block_property_test.go
@@ -1261,7 +1261,7 @@ func runBlockPropertiesBuildCmd(td *datadriven.TestData) (r *Reader, out string)
 				err = errors.Errorf("%v", r)
 			}
 		}()
-		meta, r, err = runBuildCmd(td, &opts, 0)
+		meta, r, err = runBuildCmd(td, &opts, nil /* cacheHandle */)
 	}()
 	if err != nil {
 		return r, err.Error()
diff --git a/sstable/colblk/index_block_test.go b/sstable/colblk/index_block_test.go
index c6fc7224..5c2cad04 100644
--- a/sstable/colblk/index_block_test.go
+++ b/sstable/colblk/index_block_test.go
@@ -119,17 +119,19 @@ func TestIndexIterInitHandle(t *testing.T) {
 
 	c := cache.New(10 << 10)
 	defer c.Unref()
+	ch := c.NewHandle()
+	defer ch.Close()
 
 	{
 		v := block.Alloc(len(blockData), nil)
 		copy(v.BlockData(), blockData)
 		d := (*IndexBlockDecoder)(unsafe.Pointer(v.BlockMetadata()))
 		d.Init(v.BlockData())
-		v.SetInCacheForTesting(c, cache.ID(1), base.DiskFileNum(1), 0)
+		v.SetInCacheForTesting(ch, base.DiskFileNum(1), 0)
 	}
 
 	getBlockAndIterate := func(it *IndexIter) {
-		cv := c.Get(cache.ID(1), base.DiskFileNum(1), 0)
+		cv := ch.Get(base.DiskFileNum(1), 0)
 		require.NotNil(t, cv)
 		require.NoError(t, it.InitHandle(testkeys.Comparer, block.CacheBufferHandle(cv), block.NoTransforms))
 		defer it.Close()
diff --git a/sstable/colblk/keyspan_test.go b/sstable/colblk/keyspan_test.go
index 63914b25..d008673e 100644
--- a/sstable/colblk/keyspan_test.go
+++ b/sstable/colblk/keyspan_test.go
@@ -84,14 +84,16 @@ func TestKeyspanBlockPooling(t *testing.T) {
 
 	c := cache.New(10 << 10)
 	defer c.Unref()
+	ch := c.NewHandle()
+	defer ch.Close()
 	v := block.Alloc(len(b), nil)
 	copy(v.BlockData(), b)
 	d := (*KeyspanDecoder)(unsafe.Pointer(v.BlockMetadata()))
 	d.Init(v.BlockData())
-	v.SetInCacheForTesting(c, cache.ID(1), base.DiskFileNum(1), 0)
+	v.SetInCacheForTesting(ch, base.DiskFileNum(1), 0)
 
 	getBlockAndIterate := func() {
-		cv := c.Get(cache.ID(1), base.DiskFileNum(1), 0)
+		cv := ch.Get(base.DiskFileNum(1), 0)
 		require.NotNil(t, cv)
 		it := NewKeyspanIter(testkeys.Comparer.Compare, block.CacheBufferHandle(cv), block.NoFragmentTransforms)
 		defer it.Close()
diff --git a/sstable/colblk/uints_test.go b/sstable/colblk/uints_test.go
index 1dc849a3..b82a7f79 100644
--- a/sstable/colblk/uints_test.go
+++ b/sstable/colblk/uints_test.go
@@ -49,6 +49,25 @@ func TestByteWidth(t *testing.T) {
 	}
 }
 
+func BenchmarkByteWidth(b *testing.B) {
+	for _, n := range []int{1, 2, 4, 8} {
+		b.Run(fmt.Sprintf("n=%d", n), func(b *testing.B) {
+			vals := make([]uint64, 128)
+			for i := range vals {
+				vals[i] = (1 << (8 * (n - 1))) + rand.Uint64N((1<<(8*n))-(1<<(8*(n-1))))
+			}
+			b.ResetTimer()
+			var x int
+			for i := 0; i < b.N; i++ {
+				x += int(byteWidth(vals[i&127]))
+			}
+			if x != n*b.N {
+				b.Fatal("unexpected result")
+			}
+		})
+	}
+}
+
 func TestUintEncoding(t *testing.T) {
 	for _, r := range interestingIntRanges {
 		actual := DetermineUintEncoding(r.Min, r.Max, UintEncodingRowThreshold)
@@ -92,8 +111,10 @@ func TestUintEncoding(t *testing.T) {
 		if actual != tc.expected {
 			t.Errorf("%d/%d/%d expected %s, but got %s", tc.min, tc.max, tc.numRows, tc.expected, actual)
 		}
+		if !actual.IsDelta() {
+			require.Equal(t, actual, DetermineUintEncodingNoDelta(tc.max))
+		}
 	}
-
 }
 
 func TestUints(t *testing.T) {
diff --git a/sstable/copier_test.go b/sstable/copier_test.go
index 628470d5..86b2d42f 100644
--- a/sstable/copier_test.go
+++ b/sstable/copier_test.go
@@ -25,10 +25,11 @@ import (
 func TestCopySpan(t *testing.T) {
 	fs := vfs.NewMem()
 	blockCache := cache.New(2 << 20 /* 1 MB */)
-	cacheID := cache.ID(1)
+	defer blockCache.Unref()
+	cacheHandle := blockCache.NewHandle()
+	defer cacheHandle.Close()
 	fileNameToNum := make(map[string]base.FileNum)
 	nextFileNum := base.FileNum(1)
-	defer blockCache.Unref()
 
 	keySchema := colblk.DefaultKeySchema(testkeys.Comparer, 16)
 	datadriven.RunTest(t, "testdata/copy_span", func(t *testing.T, d *datadriven.TestData) string {
@@ -110,9 +111,8 @@ func TestCopySpan(t *testing.T) {
 			rOpts := ReaderOptions{
 				ReaderOptions: block.ReaderOptions{
 					CacheOpts: sstableinternal.CacheOptions{
-						Cache:   blockCache,
-						CacheID: cacheID,
-						FileNum: base.DiskFileNum(fileNameToNum[d.CmdArgs[0].Key]),
+						CacheHandle: cacheHandle,
+						FileNum:     base.DiskFileNum(fileNameToNum[d.CmdArgs[0].Key]),
 					},
 				},
 				Comparer:   testkeys.Comparer,
@@ -164,9 +164,8 @@ func TestCopySpan(t *testing.T) {
 			rOpts := ReaderOptions{
 				ReaderOptions: block.ReaderOptions{
 					CacheOpts: sstableinternal.CacheOptions{
-						Cache:   blockCache,
-						CacheID: cacheID,
-						FileNum: base.DiskFileNum(fileNameToNum[d.CmdArgs[0].Key]),
+						CacheHandle: cacheHandle,
+						FileNum:     base.DiskFileNum(fileNameToNum[d.CmdArgs[0].Key]),
 					},
 				},
 				Comparer:   testkeys.Comparer,
diff --git a/sstable/data_test.go b/sstable/data_test.go
index 3ef693af..aaac0f12 100644
--- a/sstable/data_test.go
+++ b/sstable/data_test.go
@@ -153,20 +153,22 @@ func writeKVs(w RawWriter, input string) (err error) {
 }
 
 func runBuildCmd(
-	td *datadriven.TestData, writerOpts *WriterOptions, cacheSize int,
+	td *datadriven.TestData, writerOpts *WriterOptions, cacheHandle *cache.Handle,
 ) (*WriterMetadata, *Reader, error) {
 	meta, obj, err := runBuildMemObjCmd(td, writerOpts)
 	if err != nil {
 		return nil, nil, err
 	}
-	r, err := openReader(obj, writerOpts, cacheSize)
+	r, err := openReader(obj, writerOpts, cacheHandle)
 	if err != nil {
 		return nil, nil, err
 	}
 	return meta, r, nil
 }
 
-func openReader(obj *objstorage.MemObj, writerOpts *WriterOptions, cacheSize int) (*Reader, error) {
+func openReader(
+	obj *objstorage.MemObj, writerOpts *WriterOptions, cacheHandle *cache.Handle,
+) (*Reader, error) {
 	readerOpts := ReaderOptions{
 		Comparer:   writerOpts.Comparer,
 		KeySchemas: KeySchemas{writerOpts.KeySchema.Name: writerOpts.KeySchema},
@@ -176,11 +178,7 @@ func openReader(obj *objstorage.MemObj, writerOpts *WriterOptions, cacheSize int
 			writerOpts.FilterPolicy.Name(): writerOpts.FilterPolicy,
 		}
 	}
-	if cacheSize > 0 {
-		c := cache.New(int64(cacheSize))
-		defer c.Unref()
-		readerOpts.CacheOpts = sstableinternal.CacheOptions{Cache: c}
-	}
+	readerOpts.CacheOpts = sstableinternal.CacheOptions{CacheHandle: cacheHandle}
 	r, err := NewMemReader(obj.Data(), readerOpts)
 	if err != nil {
 		return nil, err
diff --git a/sstable/reader_test.go b/sstable/reader_test.go
index b3a7314f..b1094da5 100644
--- a/sstable/reader_test.go
+++ b/sstable/reader_test.go
@@ -176,7 +176,7 @@ func runVirtualReaderTest(t *testing.T, path string, blockSize, indexBlockSize i
 					t.Fatal(err)
 				}
 			}
-			wMeta, r, err = runBuildCmd(td, writerOpts, 0)
+			wMeta, r, err = runBuildCmd(td, writerOpts, nil /* cacheHandle */)
 			if err != nil {
 				return err.Error()
 			}
@@ -675,11 +675,24 @@ func indexLayoutString(t *testing.T, r *Reader) string {
 
 func runTestReader(t *testing.T, o WriterOptions, dir string, r *Reader, printValue bool) {
 	datadriven.Walk(t, dir, func(t *testing.T, path string) {
+		var c *cache.Cache
+		var ch *cache.Handle
+		closeCache := func() {
+			if ch != nil {
+				ch.Close()
+				ch = nil
+			}
+			if c != nil {
+				c.Unref()
+				c = nil
+			}
+		}
 		defer func() {
 			if r != nil {
 				r.Close()
 				r = nil
 			}
+			closeCache()
 		}()
 
 		datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string {
@@ -696,8 +709,11 @@ func runTestReader(t *testing.T, o WriterOptions, dir string, r *Reader, printVa
 				d.MaybeScanArgs(t, "block-size", &o.BlockSize)
 				d.MaybeScanArgs(t, "index-block-size", &o.IndexBlockSize)
 
+				closeCache()
+				c = cache.New(int64(cacheSize))
+				ch = c.NewHandle()
 				var err error
-				_, r, err = runBuildCmd(d, &o, cacheSize)
+				_, r, err = runBuildCmd(d, &o, ch)
 				if err != nil {
 					return err.Error()
 				}
@@ -894,7 +910,9 @@ func TestCompactionIteratorSetupForCompaction(t *testing.T) {
 	for _, blockSize := range blockSizes {
 		for _, indexBlockSize := range blockSizes {
 			for _, numEntries := range []uint64{0, 1, 1e5} {
-				r := buildTestTableWithProvider(t, provider, numEntries, blockSize, indexBlockSize, block.DefaultCompression, nil)
+				c := cache.New(128 << 20)
+				ch := c.NewHandle()
+				r := buildTestTableWithProvider(t, provider, numEntries, blockSize, indexBlockSize, block.DefaultCompression, nil, ch)
 				var pool block.BufferPool
 				pool.Init(5)
 				citer, err := r.NewCompactionIter(
@@ -916,6 +934,8 @@ func TestCompactionIteratorSetupForCompaction(t *testing.T) {
 				}
 				require.NoError(t, citer.Close())
 				require.NoError(t, r.Close())
+				ch.Close()
+				c.Unref()
 				pool.Release()
 			}
 		}
@@ -1721,6 +1741,7 @@ func buildTestTableWithProvider(
 	blockSize, indexBlockSize int,
 	compression block.Compression,
 	prefix []byte,
+	ch *cache.Handle,
 ) *Reader {
 	f0, _, err := provider.Create(context.Background(), base.FileTypeTable, base.DiskFileNum(0), objstorage.CreateOptions{})
 	require.NoError(t, err)
@@ -1748,12 +1769,10 @@ func buildTestTableWithProvider(
 	f1, err := provider.OpenForReading(context.Background(), base.FileTypeTable, base.DiskFileNum(0), objstorage.OpenOptions{})
 	require.NoError(t, err)
 
-	c := cache.New(128 << 20)
-	defer c.Unref()
 	r, err := NewReader(context.Background(), f1, ReaderOptions{
 		ReaderOptions: block.ReaderOptions{
 			CacheOpts: sstableinternal.CacheOptions{
-				Cache: c,
+				CacheHandle: ch,
 			},
 		},
 	})
@@ -1762,7 +1781,7 @@ func buildTestTableWithProvider(
 }
 
 func buildBenchmarkTable(
-	b *testing.B, options WriterOptions, confirmTwoLevelIndex bool, offset int,
+	b *testing.B, options WriterOptions, confirmTwoLevelIndex bool, offset int, ch *cache.Handle,
 ) (*Reader, [][]byte) {
 	mem := vfs.NewMem()
 	f0, err := mem.Create("bench", vfs.WriteCategoryUnspecified)
@@ -1791,12 +1810,10 @@ func buildBenchmarkTable(
 	if err != nil {
 		b.Fatal(err)
 	}
-	c := cache.New(128 << 20)
-	defer c.Unref()
 	r, err := newReader(f1, ReaderOptions{
 		ReaderOptions: block.ReaderOptions{
 			CacheOpts: sstableinternal.CacheOptions{
-				Cache: c,
+				CacheHandle: ch,
 			},
 		},
 	})
@@ -1839,7 +1856,9 @@ func BenchmarkTableIterSeekGE(b *testing.B) {
 	for _, bm := range basicBenchmarks {
 		b.Run(bm.name,
 			func(b *testing.B) {
-				r, keys := buildBenchmarkTable(b, bm.options, false, 0)
+				c := cache.New(128 << 20)
+				ch := c.NewHandle()
+				r, keys := buildBenchmarkTable(b, bm.options, false, 0, ch)
 				it, err := r.NewIter(NoTransforms, nil /* lower */, nil /* upper */)
 				require.NoError(b, err)
 				rng := rand.New(rand.NewPCG(0, uint64(time.Now().UnixNano())))
@@ -1852,6 +1871,8 @@ func BenchmarkTableIterSeekGE(b *testing.B) {
 				b.StopTimer()
 				it.Close()
 				r.Close()
+				ch.Close()
+				c.Unref()
 			})
 	}
 }
@@ -1860,7 +1881,9 @@ func BenchmarkTableIterSeekLT(b *testing.B) {
 	for _, bm := range basicBenchmarks {
 		b.Run(bm.name,
 			func(b *testing.B) {
-				r, keys := buildBenchmarkTable(b, bm.options, false, 0)
+				c := cache.New(128 << 20)
+				ch := c.NewHandle()
+				r, keys := buildBenchmarkTable(b, bm.options, false, 0, ch)
 				it, err := r.NewIter(NoTransforms, nil /* lower */, nil /* upper */)
 				require.NoError(b, err)
 				rng := rand.New(rand.NewPCG(0, uint64(time.Now().UnixNano())))
@@ -1873,6 +1896,8 @@ func BenchmarkTableIterSeekLT(b *testing.B) {
 				b.StopTimer()
 				it.Close()
 				r.Close()
+				ch.Close()
+				c.Unref()
 			})
 	}
 }
@@ -1881,7 +1906,9 @@ func BenchmarkTableIterNext(b *testing.B) {
 	for _, bm := range basicBenchmarks {
 		b.Run(bm.name,
 			func(b *testing.B) {
-				r, _ := buildBenchmarkTable(b, bm.options, false, 0)
+				c := cache.New(128 << 20)
+				ch := c.NewHandle()
+				r, _ := buildBenchmarkTable(b, bm.options, false, 0, ch)
 				it, err := r.NewIter(NoTransforms, nil /* lower */, nil /* upper */)
 				require.NoError(b, err)
 
@@ -1902,6 +1929,8 @@ func BenchmarkTableIterNext(b *testing.B) {
 				b.StopTimer()
 				it.Close()
 				r.Close()
+				ch.Close()
+				c.Unref()
 			})
 	}
 }
@@ -1910,7 +1939,9 @@ func BenchmarkTableIterPrev(b *testing.B) {
 	for _, bm := range basicBenchmarks {
 		b.Run(bm.name,
 			func(b *testing.B) {
-				r, _ := buildBenchmarkTable(b, bm.options, false, 0)
+				c := cache.New(128 << 20)
+				ch := c.NewHandle()
+				r, _ := buildBenchmarkTable(b, bm.options, false, 0, ch)
 				it, err := r.NewIter(NoTransforms, nil /* lower */, nil /* upper */)
 				require.NoError(b, err)
 
@@ -1931,18 +1962,24 @@ func BenchmarkTableIterPrev(b *testing.B) {
 				b.StopTimer()
 				it.Close()
 				r.Close()
+				ch.Close()
+				c.Unref()
 			})
 	}
 }
 
 func BenchmarkLayout(b *testing.B) {
-	r, _ := buildBenchmarkTable(b, WriterOptions{}, false, 0)
+	c := cache.New(128 << 20)
+	ch := c.NewHandle()
+	r, _ := buildBenchmarkTable(b, WriterOptions{}, false, 0, ch)
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		r.Layout()
 	}
 	b.StopTimer()
 	r.Close()
+	ch.Close()
+	c.Unref()
 }
 
 func BenchmarkSeqSeekGEExhausted(b *testing.B) {
@@ -1957,7 +1994,9 @@ func BenchmarkSeqSeekGEExhausted(b *testing.B) {
 			options.IndexBlockSize = 512
 		}
 		const offsetCount = 5000
-		reader, keys := buildBenchmarkTable(b, options, twoLevelIndex, offsetCount)
+		c := cache.New(128 << 20)
+		ch := c.NewHandle()
+		reader, keys := buildBenchmarkTable(b, options, twoLevelIndex, offsetCount, ch)
 		var preKeys [][]byte
 		for i := 0; i < offsetCount; i++ {
 			key := make([]byte, 8)
@@ -2024,6 +2063,8 @@ func BenchmarkSeqSeekGEExhausted(b *testing.B) {
 			}
 		}
 		reader.Close()
+		ch.Close()
+		c.Unref()
 	}
 }
 
@@ -2055,7 +2096,7 @@ func BenchmarkIteratorScanManyVersions(b *testing.B) {
 	}
 	// v2 sstable is 115,178,070 bytes. v3 sstable is 107,181,105 bytes with
 	// 99,049,269 bytes in value blocks.
-	setupBench := func(b *testing.B, tableFormat TableFormat, cacheSize int64) *Reader {
+	setupBench := func(b *testing.B, tableFormat TableFormat, ch *cache.Handle) *Reader {
 		mem := vfs.NewMem()
 		f0, err := mem.Create("bench", vfs.WriteCategoryUnspecified)
 		require.NoError(b, err)
@@ -2074,8 +2115,6 @@ func BenchmarkIteratorScanManyVersions(b *testing.B) {
 			}
 		}
 		require.NoError(b, w.Close())
-		c := cache.New(cacheSize)
-		defer c.Unref()
 		// Re-open the Filename for reading.
 		f0, err = mem.Open("bench")
 		require.NoError(b, err)
@@ -2083,7 +2122,7 @@ func BenchmarkIteratorScanManyVersions(b *testing.B) {
 			Comparer: testkeys.Comparer,
 			ReaderOptions: block.ReaderOptions{
 				CacheOpts: sstableinternal.CacheOptions{
-					Cache: c,
+					CacheHandle: ch,
 				},
 			},
 		})
@@ -2098,9 +2137,13 @@ func BenchmarkIteratorScanManyVersions(b *testing.B) {
 			for _, cacheSize := range []int64{20 << 20, 150 << 20} {
 				b.Run(fmt.Sprintf("cache-size=%s", humanize.Bytes.Int64(cacheSize)),
 					func(b *testing.B) {
-						r := setupBench(b, format, cacheSize)
+						c := cache.New(cacheSize)
+						ch := c.NewHandle()
+						r := setupBench(b, format, ch)
 						defer func() {
 							require.NoError(b, r.Close())
+							ch.Close()
+							c.Unref()
 						}()
 						b.Run("NewIter", func(b *testing.B) {
 							for i := 0; i < b.N; i++ {
@@ -2171,7 +2214,7 @@ func BenchmarkIteratorScanNextPrefix(b *testing.B) {
 	for i := 0; i < sharedPrefixLen; i++ {
 		keyBuf[i] = 'A' + byte(i)
 	}
-	setupBench := func(b *testing.B, versCount int) (r *Reader, succKeys [][]byte) {
+	setupBench := func(b *testing.B, ch *cache.Handle, versCount int) (r *Reader, succKeys [][]byte) {
 		mem := vfs.NewMem()
 		f0, err := mem.Create("bench", vfs.WriteCategoryUnspecified)
 		require.NoError(b, err)
@@ -2190,12 +2233,6 @@ func BenchmarkIteratorScanNextPrefix(b *testing.B) {
 			}
 		}
 		require.NoError(b, w.Close())
-		// NB: This 200MiB cache is sufficient for even the largest file: 10,000
-		// keys * 100 versions = 1M keys, where each key-value pair is ~140 bytes
-		// = 140MB. So we are not measuring the caching benefit of
-		// TableFormatPebblev3 storing older values in value blocks.
-		c := cache.New(200 << 20)
-		defer c.Unref()
 		// Re-open the Filename for reading.
 		f0, err = mem.Open("bench")
 		require.NoError(b, err)
@@ -2203,7 +2240,7 @@ func BenchmarkIteratorScanNextPrefix(b *testing.B) {
 			Comparer: testkeys.Comparer,
 			ReaderOptions: block.ReaderOptions{
 				CacheOpts: sstableinternal.CacheOptions{
-					Cache: c,
+					CacheHandle: ch,
 				},
 			},
 		})
@@ -2253,9 +2290,17 @@ func BenchmarkIteratorScanNextPrefix(b *testing.B) {
 	// setHasSamePrefix=false eliminates a key comparison.
 	for _, versionCount := range []int{1, 2, 10, 100} {
 		b.Run(fmt.Sprintf("versions=%d", versionCount), func(b *testing.B) {
-			r, succKeys := setupBench(b, versionCount)
+			// NB: This 200MiB cache is sufficient for even the largest file: 10,000
+			// keys * 100 versions = 1M keys, where each key-value pair is ~140 bytes
+			// = 140MB. So we are not measuring the caching benefit of
+			// TableFormatPebblev3 storing older values in value blocks.
+			c := cache.New(200 << 20)
+			ch := c.NewHandle()
+			r, succKeys := setupBench(b, ch, versionCount)
 			defer func() {
 				require.NoError(b, r.Close())
+				ch.Close()
+				c.Unref()
 			}()
 			for _, method := range []string{"seek-ge", "next-prefix"} {
 				b.Run(fmt.Sprintf("method=%s", method), func(b *testing.B) {
@@ -2334,7 +2379,7 @@ func BenchmarkIteratorScanObsolete(b *testing.B) {
 	}
 	expectedKeyCount := keys.Count()
 	keyBuf := make([]byte, keyLen)
-	setupBench := func(b *testing.B, tableFormat TableFormat, cacheSize int64) *Reader {
+	setupBench := func(b *testing.B, tableFormat TableFormat, ch *cache.Handle) *Reader {
 		mem := vfs.NewMem()
 		f0, err := mem.Create("bench", vfs.WriteCategoryUnspecified)
 		require.NoError(b, err)
@@ -2356,8 +2401,6 @@ func BenchmarkIteratorScanObsolete(b *testing.B) {
 				base.MakeInternalKey(key, 0, InternalKeyKindSet), val, forceObsolete))
 		}
 		require.NoError(b, w.Close())
-		c := cache.New(cacheSize)
-		defer c.Unref()
 		// Re-open the Filename for reading.
 		f0, err = mem.Open("bench")
 		require.NoError(b, err)
@@ -2365,7 +2408,7 @@ func BenchmarkIteratorScanObsolete(b *testing.B) {
 			Comparer: testkeys.Comparer,
 			ReaderOptions: block.ReaderOptions{
 				CacheOpts: sstableinternal.CacheOptions{
-					Cache: c,
+					CacheHandle: ch,
 				},
 			},
 		})
@@ -2378,9 +2421,13 @@ func BenchmarkIteratorScanObsolete(b *testing.B) {
 			for _, cacheSize := range []int64{1, 150 << 20} {
 				b.Run(fmt.Sprintf("cache-size=%s", humanize.Bytes.Int64(cacheSize)),
 					func(b *testing.B) {
-						r := setupBench(b, format, cacheSize)
+						c := cache.New(cacheSize)
+						ch := c.NewHandle()
+						r := setupBench(b, format, ch)
 						defer func() {
 							require.NoError(b, r.Close())
+							ch.Close()
+							c.Unref()
 						}()
 						for _, hideObsoletePoints := range []bool{false, true} {
 							b.Run(fmt.Sprintf("hide-obsolete=%t", hideObsoletePoints), func(b *testing.B) {
diff --git a/sstable/rowblk/rowblk_bench_test.go b/sstable/rowblk/rowblk_bench_test.go
index 7d47a18c..24215b5f 100644
--- a/sstable/rowblk/rowblk_bench_test.go
+++ b/sstable/rowblk/rowblk_bench_test.go
@@ -102,13 +102,13 @@ func BenchmarkBlockIterSeekGE(b *testing.B) {
 						b.ResetTimer()
 						for i := 0; i < b.N; i++ {
 							k := keys[rng.IntN(len(keys))]
-							it.SeekGE(k, base.SeekGEFlagsNone)
+							kv := it.SeekGE(k, base.SeekGEFlagsNone)
 							if testing.Verbose() {
 								if !it.Valid() && !withSyntheticSuffix {
 									b.Fatal("expected to find key")
 								}
-								if !bytes.Equal(k, it.Key().UserKey) && !withSyntheticSuffix {
-									b.Fatalf("expected %s, but found %s", k, it.Key().UserKey)
+								if !bytes.Equal(k, kv.K.UserKey) && !withSyntheticSuffix {
+									b.Fatalf("expected %s, but found %s", k, kv.K.UserKey)
 								}
 							}
 						}
@@ -144,7 +144,7 @@ func BenchmarkBlockIterSeekLT(b *testing.B) {
 						b.ResetTimer()
 						for i := 0; i < b.N; i++ {
 							j := rng.IntN(len(keys))
-							it.SeekLT(keys[j], base.SeekLTFlagsNone)
+							kv := it.SeekLT(keys[j], base.SeekLTFlagsNone)
 							if testing.Verbose() {
 								if j == 0 {
 									if it.Valid() && !withSyntheticSuffix {
@@ -155,8 +155,8 @@ func BenchmarkBlockIterSeekLT(b *testing.B) {
 										b.Fatal("expected to find key")
 									}
 									k := keys[j-1]
-									if !bytes.Equal(k, it.Key().UserKey) && !withSyntheticSuffix {
-										b.Fatalf("expected %s, but found %s", k, it.Key().UserKey)
+									if !bytes.Equal(k, kv.K.UserKey) && !withSyntheticSuffix {
+										b.Fatalf("expected %s, but found %s", k, kv.K.UserKey)
 									}
 								}
 							}
diff --git a/sstable/rowblk/rowblk_fragment_iter_test.go b/sstable/rowblk/rowblk_fragment_iter_test.go
index c995a801..933e830e 100644
--- a/sstable/rowblk/rowblk_fragment_iter_test.go
+++ b/sstable/rowblk/rowblk_fragment_iter_test.go
@@ -25,7 +25,9 @@ func TestBlockFragmentIterator(t *testing.T) {
 	comparer := testkeys.Comparer
 	var cacheVal *cache.Value
 	c := cache.New(2048)
+	cacheHandle := c.NewHandle()
 	defer func() {
+		cacheHandle.Close()
 		c.Unref()
 		if cacheVal != nil {
 			cache.Free(cacheVal)
@@ -68,13 +70,13 @@ func TestBlockFragmentIterator(t *testing.T) {
 			blockData := w.Finish()
 
 			// Evict and free the previous block.
-			c.EvictFile(1, 0)
+			cacheHandle.EvictFile(0)
 			if cacheVal != nil {
 				cache.Free(cacheVal)
 			}
 			cacheVal = cache.Alloc(block.MetadataSize + len(blockData))
 			copy(cacheVal.RawBuffer()[block.MetadataSize:], blockData)
-			c.Set(1, 0, 0, cacheVal)
+			cacheHandle.Set(0, 0, cacheVal)
 
 			for _, s := range spans {
 				buf.WriteString(s.String() + "\n")
@@ -94,7 +96,7 @@ func TestBlockFragmentIterator(t *testing.T) {
 				return d.Expected
 			}
 
-			blockHandle := block.CacheBufferHandle(c.Get(1, 0, 0))
+			blockHandle := block.CacheBufferHandle(cacheHandle.Get(0, 0))
 			require.True(t, blockHandle.Valid())
 			i, err := NewFragmentIter(0, comparer, blockHandle, transforms)
 			defer i.Close()
diff --git a/sstable/writer_test.go b/sstable/writer_test.go
index a5897c6c..18e9d244 100644
--- a/sstable/writer_test.go
+++ b/sstable/writer_test.go
@@ -151,7 +151,7 @@ func runDataDriven(t *testing.T, file string, tableFormat TableFormat, paralleli
 			if err != nil {
 				return err.Error()
 			}
-			r, err = openReader(obj, wopts, 0)
+			r, err = openReader(obj, wopts, nil /* cacheHandle */)
 			if err != nil {
 				return err.Error()
 			}
@@ -399,7 +399,7 @@ func TestWriterWithValueBlocks(t *testing.T) {
 				RequiredInPlaceValueBound: inPlaceValueBound,
 				ShortAttributeExtractor:   attributeExtractor,
 				DisableValueBlocks:        disableValueBlocks,
-			}, 0)
+			}, nil /* cacheHandle */)
 			if err != nil {
 				return err.Error()
 			}
@@ -700,12 +700,14 @@ func TestWriterClearCache(t *testing.T) {
 	// Verify that Writer clears the cache of blocks that it writes.
 	mem := vfs.NewMem()
 
+	c := cache.New(64 << 20)
+	defer c.Unref()
+
 	cacheOpts := sstableinternal.CacheOptions{
-		Cache:   cache.New(64 << 20),
-		CacheID: 1,
-		FileNum: 1,
+		CacheHandle: c.NewHandle(),
+		FileNum:     1,
 	}
-	defer cacheOpts.Cache.Unref()
+	defer cacheOpts.CacheHandle.Close()
 
 	readerOpts := ReaderOptions{
 		ReaderOptions: block.ReaderOptions{CacheOpts: cacheOpts},
@@ -775,7 +777,7 @@ func TestWriterClearCache(t *testing.T) {
 	// Poison the cache for each of the blocks.
 	poison := func(bh block.Handle) {
 		v := invalidData()
-		cacheOpts.Cache.Set(cacheOpts.CacheID, cacheOpts.FileNum, bh.Offset, v)
+		cacheOpts.CacheHandle.Set(cacheOpts.FileNum, bh.Offset, v)
 		v.Release()
 	}
 	foreachBH(layout, poison)
@@ -786,7 +788,7 @@ func TestWriterClearCache(t *testing.T) {
 
 	// Verify that the written blocks have been cleared from the cache.
 	check := func(bh block.Handle) {
-		cv := cacheOpts.Cache.Get(cacheOpts.CacheID, cacheOpts.FileNum, bh.Offset)
+		cv := cacheOpts.CacheHandle.Get(cacheOpts.FileNum, bh.Offset)
 		if cv != nil {
 			t.Fatalf("%d: expected cache to be cleared, but found %#v", bh.Offset, cv)
 		}
diff --git a/testdata/cleaner b/testdata/cleaner
index 0351b989..89bb1807 100644
--- a/testdata/cleaner
+++ b/testdata/cleaner
@@ -206,6 +206,19 @@ create: db1/000456.sst
 sync: db1/000456.sst
 close: db1/000456.sst
 
+create-bogus-file db1/000234.blob
+----
+create: db1/000234.blob
+sync: db1/000234.blob
+close: db1/000234.blob
+
+create-bogus-file db1/000345.blob
+----
+create: db1/000345.blob
+sync: db1/000345.blob
+close: db1/000345.blob
+
+
 open db1
 ----
 mkdir-all: db1 0755
@@ -249,6 +262,8 @@ rename: db1/temporary.000460.dbtmp -> db1/OPTIONS-000460
 sync: db1
 remove: db1/000123.sst
 remove: db1/000456.sst
+remove: db1/000234.blob
+remove: db1/000345.blob
 remove: db1/OPTIONS-000003
 
 list db1
diff --git a/testdata/ingest b/testdata/ingest
index 0ae0204a..3391c5b7 100644
--- a/testdata/ingest
+++ b/testdata/ingest
@@ -54,7 +54,7 @@ Virtual tables: 0 (0B)
 Local tables size: 569B
 Compression types: snappy: 1
 Block cache: 3 entries (1.1KB)  hit rate: 18.2%
-Table cache: 1 entries (920B)  hit rate: 50.0%
+Table cache: 1 entries (912B)  hit rate: 50.0%
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
 Filter utility: 0.0%
diff --git a/testdata/metrics b/testdata/metrics
index a9172d45..54b73e43 100644
--- a/testdata/metrics
+++ b/testdata/metrics
@@ -75,7 +75,7 @@ Virtual tables: 0 (0B)
 Local tables size: 589B
 Compression types: snappy: 1
 Block cache: 2 entries (716B)  hit rate: 0.0%
-Table cache: 1 entries (920B)  hit rate: 0.0%
+Table cache: 1 entries (912B)  hit rate: 0.0%
 Snapshots: 0  earliest seq num: 0
 Table iters: 1
 Filter utility: 0.0%
@@ -223,7 +223,7 @@ Virtual tables: 0 (0B)
 Local tables size: 595B
 Compression types: snappy: 1
 Block cache: 2 entries (716B)  hit rate: 33.3%
-Table cache: 1 entries (920B)  hit rate: 66.7%
+Table cache: 1 entries (912B)  hit rate: 66.7%
 Snapshots: 0  earliest seq num: 0
 Table iters: 1
 Filter utility: 0.0%
@@ -502,7 +502,7 @@ Virtual tables: 0 (0B)
 Local tables size: 4.3KB
 Compression types: snappy: 7
 Block cache: 8 entries (2.8KB)  hit rate: 9.1%
-Table cache: 1 entries (920B)  hit rate: 53.8%
+Table cache: 1 entries (912B)  hit rate: 53.8%
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
 Filter utility: 0.0%
@@ -566,7 +566,7 @@ Virtual tables: 0 (0B)
 Local tables size: 6.1KB
 Compression types: snappy: 10
 Block cache: 8 entries (2.8KB)  hit rate: 9.1%
-Table cache: 1 entries (920B)  hit rate: 53.8%
+Table cache: 1 entries (912B)  hit rate: 53.8%
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
 Filter utility: 0.0%
@@ -840,7 +840,7 @@ Virtual tables: 0 (0B)
 Local tables size: 0B
 Compression types: snappy: 1
 Block cache: 0 entries (0B)  hit rate: 0.0%
-Table cache: 1 entries (920B)  hit rate: 0.0%
+Table cache: 1 entries (912B)  hit rate: 0.0%
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
 Filter utility: 0.0%
@@ -887,7 +887,7 @@ Virtual tables: 0 (0B)
 Local tables size: 0B
 Compression types: snappy: 2
 Block cache: 4 entries (1.4KB)  hit rate: 0.0%
-Table cache: 1 entries (920B)  hit rate: 50.0%
+Table cache: 1 entries (912B)  hit rate: 50.0%
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
 Filter utility: 0.0%
@@ -935,7 +935,7 @@ Virtual tables: 0 (0B)
 Local tables size: 589B
 Compression types: snappy: 3
 Block cache: 4 entries (1.4KB)  hit rate: 0.0%
-Table cache: 1 entries (920B)  hit rate: 50.0%
+Table cache: 1 entries (912B)  hit rate: 50.0%
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
 Filter utility: 0.0%
diff --git a/testdata/version_check_consistency b/testdata/version_check_consistency
index a8ca9b3c..d741149d 100644
--- a/testdata/version_check_consistency
+++ b/testdata/version_check_consistency
@@ -13,25 +13,25 @@ check-consistency
 L0
   000005:10
 ----
-L0: 000005: file 000005 (type 2) unknown to the objstorage provider: file does not exist
+L0: 000005: file 000005 (type sstable) unknown to the objstorage provider: file does not exist
 
 check-consistency
 L0
   000001:10
 ----
-L0: 000001: file 000001 (type 2) unknown to the objstorage provider: file does not exist
+L0: 000001: file 000001 (type sstable) unknown to the objstorage provider: file does not exist
 
 check-consistency
 L0
   000001:11
 ----
-L0: 000001: file 000001 (type 2) unknown to the objstorage provider: file does not exist
+L0: 000001: file 000001 (type sstable) unknown to the objstorage provider: file does not exist
 
 check-consistency redact
 L0
   000001:11
 ----
-L0: 000001: file 000001 (type 2) unknown to the objstorage provider: file does not exist
+L0: 000001: file 000001 (type sstable) unknown to the objstorage provider: file does not exist
 
 check-consistency
 L0
@@ -41,9 +41,9 @@ L1
 L2
   000003:30
 ----
-L0: 000001: file 000001 (type 2) unknown to the objstorage provider: file does not exist
-L1: 000002: file 000002 (type 2) unknown to the objstorage provider: file does not exist
-L2: 000003: file 000003 (type 2) unknown to the objstorage provider: file does not exist
+L0: 000001: file 000001 (type sstable) unknown to the objstorage provider: file does not exist
+L1: 000002: file 000002 (type sstable) unknown to the objstorage provider: file does not exist
+L2: 000003: file 000003 (type sstable) unknown to the objstorage provider: file does not exist
 
 check-consistency
 L0
@@ -53,9 +53,9 @@ L1
 L2
   000003:33
 ----
-L0: 000001: file 000001 (type 2) unknown to the objstorage provider: file does not exist
-L1: 000002: file 000002 (type 2) unknown to the objstorage provider: file does not exist
-L2: 000003: file 000003 (type 2) unknown to the objstorage provider: file does not exist
+L0: 000001: file 000001 (type sstable) unknown to the objstorage provider: file does not exist
+L1: 000002: file 000002 (type sstable) unknown to the objstorage provider: file does not exist
+L2: 000003: file 000003 (type sstable) unknown to the objstorage provider: file does not exist
 
 check-consistency redact
 L0
@@ -65,6 +65,6 @@ L1
 L2
   000004:30
 ----
-L0: 000001: file 000001 (type 2) unknown to the objstorage provider: file does not exist
-L1: 000002: file 000002 (type 2) unknown to the objstorage provider: file does not exist
-L2: 000004: file 000004 (type 2) unknown to the objstorage provider: file does not exist
+L0: 000001: file 000001 (type sstable) unknown to the objstorage provider: file does not exist
+L1: 000002: file 000002 (type sstable) unknown to the objstorage provider: file does not exist
+L2: 000004: file 000004 (type sstable) unknown to the objstorage provider: file does not exist
diff --git a/tool/make_test_find_db.go b/tool/make_test_find_db.go
index 21f4f0b0..d8d8e2a2 100644
--- a/tool/make_test_find_db.go
+++ b/tool/make_test_find_db.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build make_test_find_db
-// +build make_test_find_db
 
 // Run using: go run -tags make_test_find_db ./tool/make_test_find_db.go
 package main
diff --git a/tool/make_test_remotecat.go b/tool/make_test_remotecat.go
index fb16accf..d35e5b28 100644
--- a/tool/make_test_remotecat.go
+++ b/tool/make_test_remotecat.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build make_test_remotecat
-// +build make_test_remotecat
 
 // Run using: go run -tags make_test_remotecat ./tool/make_test_remotecat.go
 package main
diff --git a/tool/make_test_sstables.go b/tool/make_test_sstables.go
index 9070452f..f155ca0e 100644
--- a/tool/make_test_sstables.go
+++ b/tool/make_test_sstables.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build make_test_sstables
-// +build make_test_sstables
 
 // Run using: go run -tags make_test_sstables ./tool/make_test_sstables.go
 package main
diff --git a/tool/testdata/db_excise b/tool/testdata/db_excise
index 4f903bb6..df6e39c2 100644
--- a/tool/testdata/db_excise
+++ b/tool/testdata/db_excise
@@ -52,7 +52,7 @@ scanned 2 records in 1.0s
 
 db scan testdata/broken-external-db
 ----
-L0: 000008: file 000008 (type 2) unknown to the objstorage provider: file does not exist
+L0: 000008: file 000008 (type sstable) unknown to the objstorage provider: file does not exist
 
 # The database LSM is as follows:
 #   L0.0:
diff --git a/tool/testdata/sstable_check b/tool/testdata/sstable_check
index 4e1ce235..07910aa5 100644
--- a/tool/testdata/sstable_check
+++ b/tool/testdata/sstable_check
@@ -44,14 +44,12 @@ WARNING: OUT OF ORDER KEYS!
 sstable check
 testdata/corrupted.sst
 ----
-corrupted.sst
-pebble/table: table 000000: block 87/465: crc32c checksum mismatch c8539ba5 != b972e324
+corrupted.sst: pebble/table: table 000000: block 87/465: crc32c checksum mismatch c8539ba5 != b972e324
 
 sstable check
 testdata/bad-magic.sst
 ----
-bad-magic.sst
-pebble/table: invalid table 000000: (bad magic number: 0xf6cff485b741e288)
+bad-magic.sst: pebble/table: invalid table 000000: (bad magic number: 0xf6cff485b741e288)
 
 sstable check
 ./testdata/mixed/000005.sst
diff --git a/tool/testdata/sstable_properties b/tool/testdata/sstable_properties
index 4ce51519..3e5d10be 100644
Binary files a/tool/testdata/sstable_properties and b/tool/testdata/sstable_properties differ
diff --git a/version_set_test.go b/version_set_test.go
index 271279fa..fb37bc21 100644
--- a/version_set_test.go
+++ b/version_set_test.go
@@ -76,7 +76,7 @@ func TestVersionSet(t *testing.T) {
 		// createFile only exists to prevent versionSet from complaining that a
 		// file that is becoming a zombie does not exist.
 		createFile := func(fileNum base.DiskFileNum) {
-			w, _, err := provider.Create(context.Background(), fileTypeTable, fileNum, objstorage.CreateOptions{})
+			w, _, err := provider.Create(context.Background(), base.FileTypeTable, fileNum, objstorage.CreateOptions{})
 			require.NoError(t, err)
 			require.NoError(t, w.Finish())
 		}
@@ -290,7 +290,7 @@ func TestVersionSetSeqNums(t *testing.T) {
 	var manifest vfs.File
 	for _, filename := range filenames {
 		fileType, _, ok := base.ParseFilename(mem, filename)
-		if ok && fileType == fileTypeManifest {
+		if ok && fileType == base.FileTypeManifest {
 			manifest, err = mem.Open(filename)
 			require.NoError(t, err)
 		}
diff --git a/vfs/errors_unix_test.go b/vfs/errors_unix_test.go
index 7a1c1dc4..ea441bc3 100644
--- a/vfs/errors_unix_test.go
+++ b/vfs/errors_unix_test.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build darwin || dragonfly || freebsd || linux || openbsd || netbsd
-// +build darwin dragonfly freebsd linux openbsd netbsd
 
 package vfs
 
diff --git a/vfs/syncing_file_linux_test.go b/vfs/syncing_file_linux_test.go
index 83458a3b..1b69e0ef 100644
--- a/vfs/syncing_file_linux_test.go
+++ b/vfs/syncing_file_linux_test.go
@@ -3,7 +3,6 @@
 // the LICENSE file.
 
 //go:build linux && !arm
-// +build linux,!arm
 
 package vfs