Repository: cockroachdb/pebble
Pull request #2298 (2023-02-02 16:13:53)

vfs: handle concurrent directory Syncs in disk-health checking
**db: add Options.WithFSDefaults**

Add a facility for easily layering in the default VFS middleware—currently the
disk-health checking FS. Options.EnsureDefaults by default uses the disk-health
checking FS, but most of our tests explicitly set a VFS, and in particular a
*vfs.MemFS. These tests have always run without the disk-health checking
filesystem layer. Use the new WithFSDefaults method across many Pebble unit
tests and the Pebble metamorphic tests.

This is sufficient to surface the concurrent `Sync` operations observed in
https://github.com/cockroachdb/cockroach/issues/96422 and https://github.com/cockroachdb/cockroach/issues/96414. See https://github.com/cockroachdb/pebble/pull/2282 for
context on where this panic is originating.

**vfs: handle concurrent directory Syncs in disk-health checking**

The file-level disk-health checker requires that a file not be used
concurrently, because it only supports timing a single in-flight operation at a
time. Pebble did not adhere to this contract for the data directory, which it
synced concurrently. This had the potential to leave a data directory Sync
untimed if an in-flight Syncs' timestamp was overwritten by the completion of
another Sync.

In https://github.com/cockroachdb/pebble/pull/2282 we began checking for serialized writes in `invariants` builds. This
revealed these concurrent syncs in CockroachDB test failures under `-race`:
https://github.com/cockroachdb/cockroach/issues/96414 and https://github.com/cockroachdb/cockroach/issues/96422.

cockroach-teamcity: 

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/cockroachdb/pebble/2298)
<!-- Reviewable:end -->


Patch:
diff --git a/options.go b/options.go
index 6b1b7479..c5016b06 100644
--- a/options.go
+++ b/options.go
@@ -974,14 +974,7 @@ func (o *Options) EnsureDefaults() *Options {
 	}
 
 	if o.FS == nil {
-		o.FS, o.private.fsCloser = vfs.WithDiskHealthChecks(vfs.Default, 5*time.Second,
-			func(name string, op vfs.OpType, duration time.Duration) {
-				o.EventListener.DiskSlow(DiskSlowInfo{
-					Path:     name,
-					OpType:   op,
-					Duration: duration,
-				})
-			})
+		o.WithFSDefaults()
 	}
 	if o.FlushSplitBytes <= 0 {
 		o.FlushSplitBytes = 2 * o.Levels[0].TargetFileSize
@@ -1013,6 +1006,23 @@ func (o *Options) EnsureDefaults() *Options {
 	return o
 }
 
+// WithFSDefaults configures the Options to wrap the configured filesystem with
+// the default virtual file system middleware, like disk-health checking.
+func (o *Options) WithFSDefaults() *Options {
+	if o.FS == nil {
+		o.FS = vfs.Default
+	}
+	o.FS, o.private.fsCloser = vfs.WithDiskHealthChecks(o.FS, 5*time.Second,
+		func(name string, op vfs.OpType, duration time.Duration) {
+			o.EventListener.DiskSlow(DiskSlowInfo{
+				Path:     name,
+				OpType:   op,
+				Duration: duration,
+			})
+		})
+	return o
+}
+
 // AddEventListener adds the provided event listener to the Options, in addition
 // to any existing event listener.
 func (o *Options) AddEventListener(l EventListener) {
diff --git a/vfs/disk_health.go b/vfs/disk_health.go
index b85a27be..8b6c7e54 100644
--- a/vfs/disk_health.go
+++ b/vfs/disk_health.go
@@ -280,6 +280,26 @@ func (d *diskHealthCheckingFile) timeDiskOp(opType OpType, op func()) {
 	op()
 }
 
+// diskHealthCheckingDir implements disk-health checking for directories. Unlike
+// other files, we allow directories to receive concurrent write operations
+// (Syncs are the only write operations supported by a directory.) Since the
+// diskHealthCheckingFile's timeDiskOp can only track a single in-flight
+// operation at a time, we time the operation using the filesystem-level
+// timeFilesystemOp function instead.
+type diskHealthCheckingDir struct {
+	File
+	name string
+	fs   *diskHealthCheckingFS
+}
+
+// Sync implements the io.Syncer interface.
+func (d *diskHealthCheckingDir) Sync() (err error) {
+	d.fs.timeFilesystemOp(d.name, OpTypeSync, func() {
+		err = d.File.Sync()
+	})
+	return err
+}
+
 // diskHealthCheckingFS adds disk-health checking facilities to a VFS.
 // It times disk write operations in two ways:
 //
@@ -576,11 +596,11 @@ func (d *diskHealthCheckingFS) OpenDir(name string) (File, error) {
 	}
 	// Directories opened with OpenDir must be opened with health checking,
 	// because they may be explicitly synced.
-	checkingFile := newDiskHealthCheckingFile(f, d.diskSlowThreshold, func(opType OpType, duration time.Duration) {
-		d.onSlowDisk(name, opType, duration)
-	})
-	checkingFile.startTicker()
-	return checkingFile, nil
+	return &diskHealthCheckingDir{
+		File: f,
+		name: name,
+		fs:   d,
+	}, nil
 }
 
 // PathBase implements the FS interface.


Test patch:
diff --git a/cleaner_test.go b/cleaner_test.go
index a7784923..0dc91527 100644
--- a/cleaner_test.go
+++ b/cleaner_test.go
@@ -25,11 +25,11 @@ func TestArchiveCleaner(t *testing.T) {
 
 	var buf syncedBuffer
 	mem := vfs.NewMem()
-	opts := &Options{
+	opts := (&Options{
 		Cleaner: ArchiveCleaner{},
 		FS:      loggingFS{mem, &buf},
 		WALDir:  "wal",
-	}
+	}).WithFSDefaults()
 
 	datadriven.RunTest(t, "testdata/cleaner", func(t *testing.T, td *datadriven.TestData) string {
 		switch td.Cmd {
diff --git a/compaction_test.go b/compaction_test.go
index 881ccc30..46d7e7e0 100644
--- a/compaction_test.go
+++ b/compaction_test.go
@@ -534,8 +534,7 @@ func TestPickCompaction(t *testing.T) {
 }
 
 func TestElideTombstone(t *testing.T) {
-	opts := &Options{}
-	opts.EnsureDefaults()
+	opts := (&Options{}).EnsureDefaults().WithFSDefaults()
 
 	newFileMeta := func(smallest, largest base.InternalKey) *fileMetadata {
 		m := (&fileMetadata{}).ExtendPointKeyBounds(opts.Comparer.Compare, smallest, largest)
@@ -917,9 +916,7 @@ func (t *cpuPermissionGranter) CPUWorkDone(_ CPUWorkHandle) {
 // the acquired handles are returned.
 func TestCompactionCPUGranter(t *testing.T) {
 	mem := vfs.NewMem()
-	opts := &Options{
-		FS: mem,
-	}
+	opts := (&Options{FS: mem}).WithFSDefaults()
 	g := &cpuPermissionGranter{permit: true}
 	opts.Experimental.CPUWorkPermissionGranter = g
 	d, err := Open("", opts)
@@ -940,9 +937,7 @@ func TestCompactionCPUGranter(t *testing.T) {
 // Tests that there's no errors or panics when the default CPU granter is used.
 func TestCompactionCPUGranterDefault(t *testing.T) {
 	mem := vfs.NewMem()
-	opts := &Options{
-		FS: mem,
-	}
+	opts := (&Options{FS: mem}).WithFSDefaults()
 	d, err := Open("", opts)
 	if err != nil {
 		t.Fatalf("Open: %v", err)
@@ -970,7 +965,7 @@ func TestCompaction(t *testing.T) {
 		DebugCheck:            DebugCheckLevels,
 		L0CompactionThreshold: 8,
 	}
-	opts.testingRandomized()
+	opts.testingRandomized().WithFSDefaults()
 	d, err := Open("", opts)
 	if err != nil {
 		t.Fatalf("Open: %v", err)
@@ -1275,11 +1270,11 @@ func TestManualCompaction(t *testing.T) {
 		mem = vfs.NewMem()
 		require.NoError(t, mem.MkdirAll("ext", 0755))
 
-		opts := &Options{
+		opts := (&Options{
 			FS:                 mem,
 			DebugCheck:         DebugCheckLevels,
 			FormatMajorVersion: randVersion(minVersion, maxVersion),
-		}
+		}).WithFSDefaults()
 		opts.DisableAutomaticCompactions = true
 
 		var err error
@@ -1373,12 +1368,12 @@ func TestManualCompaction(t *testing.T) {
 				}
 
 				mem = vfs.NewMem()
-				opts := &Options{
+				opts := (&Options{
 					FS:                          mem,
 					DebugCheck:                  DebugCheckLevels,
 					FormatMajorVersion:          randVersion(minVersion, maxVersion),
 					DisableAutomaticCompactions: true,
-				}
+				}).WithFSDefaults()
 
 				var err error
 				if d, err = runDBDefineCmd(td, opts); err != nil {
@@ -1905,7 +1900,7 @@ func TestCompactionDeleteOnlyHints(t *testing.T) {
 				return nil, err
 			}
 		}
-		opts := &Options{
+		opts := (&Options{
 			FS:         vfs.NewMem(),
 			DebugCheck: DebugCheckLevels,
 			EventListener: &EventListener{
@@ -1917,7 +1912,7 @@ func TestCompactionDeleteOnlyHints(t *testing.T) {
 				},
 			},
 			FormatMajorVersion: FormatNewest,
-		}
+		}).WithFSDefaults()
 
 		// Collection of table stats can trigger compactions. As we want full
 		// control over when compactions are run, disable stats by default.
@@ -2175,7 +2170,7 @@ func TestCompactionTombstones(t *testing.T) {
 						return err.Error()
 					}
 				}
-				opts := &Options{
+				opts := (&Options{
 					FS:         vfs.NewMem(),
 					DebugCheck: DebugCheckLevels,
 					EventListener: &EventListener{
@@ -2184,7 +2179,7 @@ func TestCompactionTombstones(t *testing.T) {
 						},
 					},
 					FormatMajorVersion: FormatNewest,
-				}
+				}).WithFSDefaults()
 				var err error
 				d, err = runDBDefineCmd(td, opts)
 				if err != nil {
@@ -2383,7 +2378,7 @@ func TestCompactionReadTriggered(t *testing.T) {
 						return err.Error()
 					}
 				}
-				opts := &Options{
+				opts := (&Options{
 					FS:         vfs.NewMem(),
 					DebugCheck: DebugCheckLevels,
 					EventListener: &EventListener{
@@ -2391,7 +2386,7 @@ func TestCompactionReadTriggered(t *testing.T) {
 							compactInfo = &info
 						},
 					},
-				}
+				}).WithFSDefaults()
 				var err error
 				d, err = runDBDefineCmd(td, opts)
 				if err != nil {
@@ -2853,7 +2848,7 @@ func TestCompactionErrorCleanup(t *testing.T) {
 
 	mem := vfs.NewMem()
 	ii := errorfs.OnIndex(math.MaxInt32) // start disabled
-	opts := &Options{
+	opts := (&Options{
 		FS:     errorfs.Wrap(mem, ii),
 		Levels: make([]LevelOptions, numLevels),
 		EventListener: &EventListener{
@@ -2871,7 +2866,7 @@ func TestCompactionErrorCleanup(t *testing.T) {
 				}
 			},
 		},
-	}
+	}).WithFSDefaults()
 	for i := range opts.Levels {
 		opts.Levels[i].TargetFileSize = 1
 	}
@@ -3157,7 +3152,7 @@ func TestFlushInvariant(t *testing.T) {
 							},
 						},
 						DebugCheck: DebugCheckLevels,
-					}))
+					}).WithFSDefaults())
 					require.NoError(t, err)
 
 					require.NoError(t, d.Set([]byte("hello"), nil, NoSync))
@@ -3208,7 +3203,7 @@ func TestCompactFlushQueuedMemTableAndFlushMetrics(t *testing.T) {
 	mem := vfs.NewMem()
 	d, err := Open("", testingRandomized(&Options{
 		FS: mem,
-	}))
+	}).WithFSDefaults())
 	require.NoError(t, err)
 
 	// Add the key "a" to the memtable, then fill up the memtable with the key
@@ -3267,7 +3262,7 @@ func TestCompactFlushQueuedLargeBatch(t *testing.T) {
 	mem := vfs.NewMem()
 	d, err := Open("", testingRandomized(&Options{
 		FS: mem,
-	}))
+	}).WithFSDefaults())
 	require.NoError(t, err)
 
 	// The default large batch threshold is slightly less than 1/2 of the
@@ -3301,7 +3296,7 @@ func TestCompactFlushQueuedLargeBatch(t *testing.T) {
 func TestCleanerCond(t *testing.T) {
 	d, err := Open("", testingRandomized(&Options{
 		FS: vfs.NewMem(),
-	}))
+	}).WithFSDefaults())
 	require.NoError(t, err)
 
 	for i := 0; i < 10; i++ {
@@ -3359,7 +3354,7 @@ func TestFlushError(t *testing.T) {
 				t.Log(err)
 			},
 		},
-	}))
+	}).WithFSDefaults())
 	require.NoError(t, err)
 	require.NoError(t, d.Set([]byte("a"), []byte("foo"), NoSync))
 	require.NoError(t, d.Flush())
@@ -3406,7 +3401,7 @@ func TestAdjustGrandparentOverlapBytesForFlush(t *testing.T) {
 func TestCompactionInvalidBounds(t *testing.T) {
 	db, err := Open("", testingRandomized(&Options{
 		FS: vfs.NewMem(),
-	}))
+	}).WithFSDefaults())
 	require.NoError(t, err)
 	defer db.Close()
 	require.NoError(t, db.Compact([]byte("a"), []byte("b"), false))
@@ -3654,7 +3649,7 @@ func TestMarkedForCompaction(t *testing.T) {
 	}()
 
 	var buf bytes.Buffer
-	opts := &Options{
+	opts := (&Options{
 		FS:                          mem,
 		DebugCheck:                  DebugCheckLevels,
 		DisableAutomaticCompactions: true,
@@ -3668,7 +3663,7 @@ func TestMarkedForCompaction(t *testing.T) {
 				fmt.Fprintln(&buf, info)
 			},
 		},
-	}
+	}).WithFSDefaults()
 
 	reset := func() {
 		if d != nil {
@@ -3836,7 +3831,7 @@ func TestCompaction_LogAndApplyFails(t *testing.T) {
 		var db *DB
 		inj := &createManifestErrorInjector{}
 		logger := &fatalCapturingLogger{}
-		opts := &Options{
+		opts := (&Options{
 			FS: errorfs.Wrap(vfs.NewMem(), inj),
 			// Rotate the manifest after each write. This is required to trigger a
 			// file creation, into which errors can be injected.
@@ -3850,7 +3845,7 @@ func TestCompaction_LogAndApplyFails(t *testing.T) {
 				},
 			},
 			DisableAutomaticCompactions: true,
-		}
+		}).WithFSDefaults()
 
 		db, err := Open("", opts)
 		require.NoError(t, err)
diff --git a/format_major_version_test.go b/format_major_version_test.go
index 24edc847..ce24c0ae 100644
--- a/format_major_version_test.go
+++ b/format_major_version_test.go
@@ -32,7 +32,7 @@ func TestFormatMajorVersion_MigrationDefined(t *testing.T) {
 
 func TestRatchetFormat(t *testing.T) {
 	fs := vfs.NewMem()
-	d, err := Open("", &Options{FS: fs})
+	d, err := Open("", (&Options{FS: fs}).WithFSDefaults())
 	require.NoError(t, err)
 	require.NoError(t, d.Set([]byte("foo"), []byte("bar"), Sync))
 	require.Equal(t, FormatMostCompatible, d.FormatMajorVersion())
@@ -63,7 +63,7 @@ func TestRatchetFormat(t *testing.T) {
 
 	// If we Open the database again, leaving the default format, the
 	// database should Open using the persisted FormatNewest.
-	d, err = Open("", &Options{FS: fs})
+	d, err = Open("", (&Options{FS: fs}).WithFSDefaults())
 	require.NoError(t, err)
 	require.Equal(t, FormatNewest, d.FormatMajorVersion())
 	require.NoError(t, d.Close())
@@ -74,10 +74,10 @@ func TestRatchetFormat(t *testing.T) {
 	require.NoError(t, m.Move("999999"))
 	require.NoError(t, m.Close())
 
-	_, err = Open("", &Options{
+	_, err = Open("", (&Options{
 		FS:                 fs,
 		FormatMajorVersion: FormatVersioned,
-	})
+	}).WithFSDefaults())
 	require.Error(t, err)
 	require.EqualError(t, err, `pebble: database "" written in format major version 999999`)
 }
@@ -108,10 +108,10 @@ func TestFormatMajorVersions(t *testing.T) {
 	for vers := FormatMostCompatible; vers <= FormatNewest; vers++ {
 		t.Run(fmt.Sprintf("vers=%03d", vers), func(t *testing.T) {
 			fs := vfs.NewStrictMem()
-			opts := &Options{
+			opts := (&Options{
 				FS:                 fs,
 				FormatMajorVersion: vers,
-			}
+			}).WithFSDefaults()
 
 			// Create a database at this format major version and perform
 			// some very basic operations.
@@ -255,7 +255,7 @@ func TestSplitUserKeyMigration(t *testing.T) {
 					}
 					buf.Reset()
 				}
-				opts = &Options{
+				opts = (&Options{
 					FormatMajorVersion: FormatBlockPropertyCollector,
 					EventListener: &EventListener{
 						CompactionEnd: func(info CompactionInfo) {
@@ -267,7 +267,7 @@ func TestSplitUserKeyMigration(t *testing.T) {
 						},
 					},
 					DisableAutomaticCompactions: true,
-				}
+				}).WithFSDefaults()
 				var err error
 				if d, err = runDBDefineCmd(td, opts); err != nil {
 					return err.Error()
@@ -363,10 +363,10 @@ func TestPebblev1Migration(t *testing.T) {
 						return fmt.Sprintf("unknown argument: %s", cmd)
 					}
 				}
-				opts := &Options{
+				opts := (&Options{
 					FS:                 vfs.NewMem(),
 					FormatMajorVersion: FormatMajorVersion(version),
-				}
+				}).WithFSDefaults()
 				d, err = Open("", opts)
 				if err != nil {
 					return err.Error()
@@ -486,13 +486,13 @@ func TestPebblev1MigrationRace(t *testing.T) {
 	defer cache.Unref()
 	tableCache := NewTableCache(cache, 1, 5)
 	defer tableCache.Unref()
-	d, err := Open("", &Options{
+	d, err := Open("", (&Options{
 		Cache:              cache,
 		FS:                 vfs.NewMem(),
 		FormatMajorVersion: FormatMajorVersion(FormatPrePebblev1Marked - 1),
 		TableCache:         tableCache,
 		Levels:             []LevelOptions{{TargetFileSize: 1}},
-	})
+	}).WithFSDefaults())
 	require.NoError(t, err)
 	defer d.Close()
 
@@ -531,7 +531,7 @@ func TestPebblev1MigrationRace(t *testing.T) {
 // Regression test for #2044, where multiple concurrent compactions can lead
 // to an indefinite wait on the compaction goroutine in compactMarkedFilesLocked.
 func TestPebblev1MigrationConcurrencyRace(t *testing.T) {
-	opts := &Options{
+	opts := (&Options{
 		Comparer:           testkeys.Comparer,
 		FS:                 vfs.NewMem(),
 		FormatMajorVersion: FormatSplitUserKeysMarked,
@@ -539,7 +539,7 @@ func TestPebblev1MigrationConcurrencyRace(t *testing.T) {
 		MaxConcurrentCompactions: func() int {
 			return 4
 		},
-	}
+	}).WithFSDefaults()
 	func() {
 		d, err := Open("", opts)
 		require.NoError(t, err)
diff --git a/ingest_test.go b/ingest_test.go
index 15800f0f..c1fbed6b 100644
--- a/ingest_test.go
+++ b/ingest_test.go
@@ -112,10 +112,10 @@ func TestIngestLoad(t *testing.T) {
 			}
 			w.Close()
 
-			opts := &Options{
+			opts := (&Options{
 				Comparer: DefaultComparer,
 				FS:       mem,
-			}
+			}).WithFSDefaults()
 			meta, _, err := ingestLoad(opts, dbVersion, []string{"ext"}, 0, []FileNum{1})
 			if err != nil {
 				return err.Error()
@@ -198,10 +198,10 @@ func TestIngestLoadRand(t *testing.T) {
 		}()
 	}
 
-	opts := &Options{
+	opts := (&Options{
 		Comparer: DefaultComparer,
 		FS:       mem,
-	}
+	}).WithFSDefaults()
 	meta, _, err := ingestLoad(opts, version, paths, 0, pending)
 	require.NoError(t, err)
 
@@ -219,10 +219,10 @@ func TestIngestLoadInvalid(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, f.Close())
 
-	opts := &Options{
+	opts := (&Options{
 		Comparer: DefaultComparer,
 		FS:       mem,
-	}
+	}).WithFSDefaults()
 	if _, _, err := ingestLoad(opts, FormatNewest, []string{"invalid"}, 0, []FileNum{1}); err == nil {
 		t.Fatalf("expected error, but found success")
 	}
@@ -287,11 +287,10 @@ func TestIngestLink(t *testing.T) {
 	const count = 10
 	for i := 0; i <= count; i++ {
 		t.Run("", func(t *testing.T) {
-			mem := vfs.NewMem()
-			opts := &Options{FS: mem}
-			opts.EnsureDefaults()
-			require.NoError(t, mem.MkdirAll(dir, 0755))
-			objProvider := objstorage.New(objstorage.DefaultSettings(mem, dir))
+			opts := &Options{FS: vfs.NewMem()}
+			opts.EnsureDefaults().WithFSDefaults()
+			require.NoError(t, opts.FS.MkdirAll(dir, 0755))
+			objProvider := objstorage.New(objstorage.DefaultSettings(opts.FS, dir))
 
 			paths := make([]string, 10)
 			meta := make([]*fileMetadata, len(paths))
@@ -300,7 +299,7 @@ func TestIngestLink(t *testing.T) {
 				paths[j] = fmt.Sprintf("external%d", j)
 				meta[j] = &fileMetadata{}
 				meta[j].FileNum = FileNum(j)
-				f, err := mem.Create(paths[j])
+				f, err := opts.FS.Create(paths[j])
 				require.NoError(t, err)
 
 				contents[j] = []byte(fmt.Sprintf("data%d", j))
@@ -312,7 +311,7 @@ func TestIngestLink(t *testing.T) {
 			}
 
 			if i < count {
-				mem.Remove(paths[i])
+				opts.FS.Remove(paths[i])
 			}
 
 			_, err := ingestLink(0 /* jobID */, opts, objProvider, paths, meta)
@@ -324,7 +323,7 @@ func TestIngestLink(t *testing.T) {
 				require.NoError(t, err)
 			}
 
-			files, err := mem.List(dir)
+			files, err := opts.FS.List(dir)
 			require.NoError(t, err)
 
 			sort.Strings(files)
@@ -339,7 +338,7 @@ func TestIngestLink(t *testing.T) {
 					t.Fatalf("expected %d files, but found:\n%s", count, strings.Join(files, "\n"))
 				}
 				for j := range files {
-					ftype, fileNum, ok := base.ParseFilename(mem, files[j])
+					ftype, fileNum, ok := base.ParseFilename(opts.FS, files[j])
 					if !ok {
 						t.Fatalf("unable to parse filename: %s", files[j])
 					}
@@ -349,7 +348,7 @@ func TestIngestLink(t *testing.T) {
 					if FileNum(j) != fileNum {
 						t.Fatalf("expected table %d, but found %d", j, fileNum)
 					}
-					f, err := mem.Open(mem.PathJoin(dir, files[j]))
+					f, err := opts.FS.Open(opts.FS.PathJoin(dir, files[j]))
 					require.NoError(t, err)
 
 					data, err := io.ReadAll(f)
@@ -372,7 +371,7 @@ func TestIngestLinkFallback(t *testing.T) {
 	require.NoError(t, err)
 
 	opts := &Options{FS: errorfs.Wrap(mem, errorfs.OnIndex(0))}
-	opts.EnsureDefaults()
+	opts.EnsureDefaults().WithFSDefaults()
 	objProvider := objstorage.New(objstorage.DefaultSettings(opts.FS, ""))
 
 	meta := []*fileMetadata{{FileNum: 1}}
@@ -420,14 +419,14 @@ func TestOverlappingIngestedSSTs(t *testing.T) {
 		}
 
 		require.NoError(t, mem.MkdirAll("ext", 0755))
-		opts = &Options{
+		opts = (&Options{
 			FS:                          mem,
 			MemTableStopWritesThreshold: 4,
 			L0CompactionThreshold:       100,
 			L0StopWritesThreshold:       100,
 			DebugCheck:                  DebugCheckLevels,
 			FormatMajorVersion:          FormatNewest,
-		}
+		}).WithFSDefaults()
 		// Disable automatic compactions because otherwise we'll race with
 		// delete-only compactions triggered by ingesting range tombstones.
 		opts.DisableAutomaticCompactions = true
@@ -614,7 +613,7 @@ func TestIngestMemtableOverlaps(t *testing.T) {
 					opts := &Options{
 						Comparer: &comparer,
 					}
-					opts.EnsureDefaults()
+					opts.EnsureDefaults().WithFSDefaults()
 					if len(d.CmdArgs) > 1 {
 						return fmt.Sprintf("%s expects at most 1 argument", d.Cmd)
 					}
@@ -743,6 +742,7 @@ func TestIngestTargetLevel(t *testing.T) {
 			opts := Options{
 				FormatMajorVersion: FormatNewest,
 			}
+			opts.WithFSDefaults()
 			if d, err = runDBDefineCmd(td, &opts); err != nil {
 				return err.Error()
 			}
diff --git a/internal/metamorphic/meta_test.go b/internal/metamorphic/meta_test.go
index b03617d2..150b14a8 100644
--- a/internal/metamorphic/meta_test.go
+++ b/internal/metamorphic/meta_test.go
@@ -163,6 +163,8 @@ func testMetaRun(t *testing.T, runDir string, seed uint64, historyPath string) {
 			opts.FS = vfs.NewMem()
 		}
 	}
+	opts.WithFSDefaults()
+
 	threads := testOpts.threads
 	if *maxThreads < threads {
 		threads = *maxThreads
diff --git a/iterator_histories_test.go b/iterator_histories_test.go
index ea6298c8..80bfc94d 100644
--- a/iterator_histories_test.go
+++ b/iterator_histories_test.go
@@ -52,6 +52,7 @@ func TestIterHistories(t *testing.T) {
 			}
 			opts.DisableAutomaticCompactions = true
 			opts.EnsureDefaults()
+			opts.WithFSDefaults()
 
 			for _, cmdArg := range td.CmdArgs {
 				switch cmdArg.Key {