Repository: cockroachdb/pebble
Pull request #3848 (2024-08-13 19:00:29)

db: refactor disk usage estimate using range annotations
This change updates `db.EstimateDiskUsage` to use range annotations to
estimate the disk usage of a key range. This should improve the
performance of repeated disk usage estimates for similar or identical
key ranges.

At the Cockroach layer we use `db.EstimateDiskUsage` in a few places,
most notably when [computing MVCC span stats](https://github.com/cockroachdb/cockroach/blob/master/pkg/server/span_stats_server.go#L217).

Informs: cockroachdb#3793


cockroach-teamcity: 

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/cockroachdb/pebble/3848)
<!-- Reviewable:end -->


Patch:
diff --git a/db.go b/db.go
index 8c7b6f16..0fca0798 100644
--- a/db.go
+++ b/db.go
@@ -501,6 +501,14 @@ type DB struct {
 			// validating is set to true when validation is running.
 			validating bool
 		}
+
+		// annotators contains various instances of manifest.Annotator which
+		// should be protected from concurrent access.
+		annotators struct {
+			totalSize    *manifest.Annotator[uint64]
+			remoteSize   *manifest.Annotator[uint64]
+			externalSize *manifest.Annotator[uint64]
+		}
 	}
 
 	// Normally equal to time.Now() but may be overridden in tests.
@@ -2228,6 +2236,31 @@ func (d *DB) SSTables(opts ...SSTablesOption) ([][]SSTableInfo, error) {
 	return destLevels, nil
 }
 
+// makeFileSizeAnnotator returns an annotator that computes the total size of
+// files that meet some criteria defined by filter.
+func (d *DB) makeFileSizeAnnotator(filter func(f *fileMetadata) bool) *manifest.Annotator[uint64] {
+	return &manifest.Annotator[uint64]{
+		Aggregator: manifest.SumAggregator{
+			AccumulateFunc: func(f *fileMetadata) (uint64, bool) {
+				if filter(f) {
+					return f.Size, true
+				}
+				return 0, true
+			},
+			AccumulatePartialOverlapFunc: func(f *fileMetadata, bounds base.UserKeyBounds) uint64 {
+				if filter(f) {
+					size, err := d.tableCache.estimateSize(f, bounds.Start, bounds.End.Key)
+					if err != nil {
+						return 0
+					}
+					return size
+				}
+				return 0
+			},
+		},
+	}
+}
+
 // EstimateDiskUsage returns the estimated filesystem space used in bytes for
 // storing the range `[start, end]`. The estimation is computed as follows:
 //
@@ -2254,7 +2287,9 @@ func (d *DB) EstimateDiskUsageByBackingType(
 	if err := d.closed.Load(); err != nil {
 		panic(err)
 	}
-	if d.opts.Comparer.Compare(start, end) > 0 {
+
+	bounds := base.UserKeyBoundsInclusive(start, end)
+	if !bounds.Valid(d.cmp) {
 		return 0, 0, 0, errors.New("invalid key-range specified (start > end)")
 	}
 
@@ -2264,70 +2299,13 @@ func (d *DB) EstimateDiskUsageByBackingType(
 	readState := d.loadReadState()
 	defer readState.unref()
 
-	for level, files := range readState.current.Levels {
-		iter := files.Iter()
-		if level > 0 {
-			// We can only use `Overlaps` to restrict `files` at L1+ since at L0 it
-			// expands the range iteratively until it has found a set of files that
-			// do not overlap any other L0 files outside that set.
-			overlaps := readState.current.Overlaps(level, base.UserKeyBoundsInclusive(start, end))
-			iter = overlaps.Iter()
-		}
-		for file := iter.First(); file != nil; file = iter.Next() {
-			if d.opts.Comparer.Compare(start, file.Smallest.UserKey) <= 0 &&
-				d.opts.Comparer.Compare(file.Largest.UserKey, end) <= 0 {
-				// The range fully contains the file, so skip looking it up in
-				// table cache/looking at its indexes, and add the full file size.
-				meta, err := d.objProvider.Lookup(fileTypeTable, file.FileBacking.DiskFileNum)
-				if err != nil {
-					return 0, 0, 0, err
-				}
-				if meta.IsRemote() {
-					remoteSize += file.Size
-					if meta.Remote.CleanupMethod == objstorage.SharedNoCleanup {
-						externalSize += file.Size
-					}
-				}
-				totalSize += file.Size
-			} else if d.opts.Comparer.Compare(file.Smallest.UserKey, end) <= 0 &&
-				d.opts.Comparer.Compare(start, file.Largest.UserKey) <= 0 {
-				var size uint64
-				var err error
-				if file.Virtual {
-					err = d.tableCache.withVirtualReader(
-						file.VirtualMeta(),
-						func(r sstable.VirtualReader) (err error) {
-							size, err = r.EstimateDiskUsage(start, end)
-							return err
-						},
-					)
-				} else {
-					err = d.tableCache.withReader(
-						file.PhysicalMeta(),
-						func(r *sstable.Reader) (err error) {
-							size, err = r.EstimateDiskUsage(start, end)
-							return err
-						},
-					)
-				}
-				if err != nil {
-					return 0, 0, 0, err
-				}
-				meta, err := d.objProvider.Lookup(fileTypeTable, file.FileBacking.DiskFileNum)
-				if err != nil {
-					return 0, 0, 0, err
-				}
-				if meta.IsRemote() {
-					remoteSize += size
-					if meta.Remote.CleanupMethod == objstorage.SharedNoCleanup {
-						externalSize += size
-					}
-				}
-				totalSize += size
-			}
-		}
-	}
-	return totalSize, remoteSize, externalSize, nil
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	totalSize = *d.mu.annotators.totalSize.VersionRangeAnnotation(readState.current, bounds)
+	remoteSize = *d.mu.annotators.remoteSize.VersionRangeAnnotation(readState.current, bounds)
+	externalSize = *d.mu.annotators.externalSize.VersionRangeAnnotation(readState.current, bounds)
+	return
 }
 
 func (d *DB) walPreallocateSize() int {
diff --git a/internal/manifest/annotator.go b/internal/manifest/annotator.go
index 5e63fb4e..06be66fe 100644
--- a/internal/manifest/annotator.go
+++ b/internal/manifest/annotator.go
@@ -59,6 +59,14 @@ type AnnotationAggregator[T any] interface {
 	Merge(src *T, dst *T) *T
 }
 
+// A PartialOverlapAnnotationAggregator is an extension of AnnotationAggregator
+// that allows for custom accumulation of range annotations for files that only
+// partially overlap with the range.
+type PartialOverlapAnnotationAggregator[T any] interface {
+	AnnotationAggregator[T]
+	AccumulatePartialOverlap(f *FileMetadata, dst *T, bounds base.UserKeyBounds) *T
+}
+
 type annotation struct {
 	// annotator is a pointer to the Annotator that computed this annotation.
 	// NB: This is untyped to allow AnnotationAggregator to use Go generics,
@@ -165,6 +173,15 @@ func (a *Annotator[T]) accumulateRangeAnnotation(
 
 	// Accumulate annotations from every item that overlaps the bounds.
 	for i := leftItem; i < rightItem; i++ {
+		if i == leftItem || i == rightItem-1 {
+			if agg, ok := a.Aggregator.(PartialOverlapAnnotationAggregator[T]); ok {
+				fb := n.items[i].UserKeyBounds()
+				if cmp(bounds.Start, fb.Start) > 0 || bounds.End.CompareUpperBounds(cmp, fb.End) < 0 {
+					a.scratch = agg.AccumulatePartialOverlap(n.items[i], a.scratch, bounds)
+					continue
+				}
+			}
+		}
 		v, _ := a.Aggregator.Accumulate(n.items[i], a.scratch)
 		a.scratch = v
 	}
@@ -258,6 +275,26 @@ func (a *Annotator[T]) LevelRangeAnnotation(lm LevelMetadata, bounds base.UserKe
 	return a.scratch
 }
 
+// VersionRangeAnnotation calculates the annotation defined by this Annotator
+// for all files within the given Version which are within the range
+// defined by bounds.
+func (a *Annotator[T]) VersionRangeAnnotation(v *Version, bounds base.UserKeyBounds) *T {
+	accumulateSlice := func(ls LevelSlice) {
+		if ls.Empty() {
+			return
+		}
+		a.accumulateRangeAnnotation(ls.iter.r, v.cmp.Compare, bounds, false, false)
+	}
+	a.scratch = a.Aggregator.Zero(a.scratch)
+	for _, ls := range v.L0SublevelFiles {
+		accumulateSlice(ls)
+	}
+	for _, lm := range v.Levels[1:] {
+		accumulateSlice(lm.Slice())
+	}
+	return a.scratch
+}
+
 // InvalidateAnnotation clears any cached annotations defined by Annotator. A
 // pointer to the Annotator is used as the key for pre-calculated values, so
 // the same Annotator must be used to clear the appropriate cached annotation.
@@ -270,13 +307,15 @@ func (a *Annotator[T]) InvalidateLevelAnnotation(lm LevelMetadata) {
 	a.invalidateNodeAnnotation(lm.tree.root)
 }
 
-// sumAggregator defines an Aggregator which sums together a uint64 value
+// SumAggregator defines an Aggregator which sums together a uint64 value
 // across files.
-type sumAggregator struct {
-	accumulateFunc func(f *FileMetadata) (v uint64, cacheOK bool)
+type SumAggregator struct {
+	AccumulateFunc               func(f *FileMetadata) (v uint64, cacheOK bool)
+	AccumulatePartialOverlapFunc func(f *FileMetadata, bounds base.UserKeyBounds) uint64
 }
 
-func (sa sumAggregator) Zero(dst *uint64) *uint64 {
+// Zero implements AnnotationAggregator.Zero, returning a new uint64 set to 0.
+func (sa SumAggregator) Zero(dst *uint64) *uint64 {
 	if dst == nil {
 		return new(uint64)
 	}
@@ -284,13 +323,31 @@ func (sa sumAggregator) Zero(dst *uint64) *uint64 {
 	return dst
 }
 
-func (sa sumAggregator) Accumulate(f *FileMetadata, dst *uint64) (v *uint64, cacheOK bool) {
-	accumulated, ok := sa.accumulateFunc(f)
+// Accumulate implements AnnotationAggregator.Accumulate, accumulating a single
+// file's uint64 value.
+func (sa SumAggregator) Accumulate(f *FileMetadata, dst *uint64) (v *uint64, cacheOK bool) {
+	accumulated, ok := sa.AccumulateFunc(f)
 	*dst += accumulated
 	return dst, ok
 }
 
-func (sa sumAggregator) Merge(src *uint64, dst *uint64) *uint64 {
+// AccumulatePartialOverlap implements
+// PartialOverlapAnnotationAggregator.AccumulatePartialOverlap, accumulating a
+// single file's uint64 value for a file which only partially overlaps with the
+// range defined by bounds.
+func (sa SumAggregator) AccumulatePartialOverlap(
+	f *FileMetadata, dst *uint64, bounds base.UserKeyBounds,
+) *uint64 {
+	if sa.AccumulatePartialOverlapFunc == nil {
+		v, _ := sa.Accumulate(f, dst)
+		return v
+	}
+	*dst += sa.AccumulatePartialOverlapFunc(f, bounds)
+	return dst
+}
+
+// Merge implements AnnotationAggregator.Merge by summing two uint64 values.
+func (sa SumAggregator) Merge(src *uint64, dst *uint64) *uint64 {
 	*dst += *src
 	return dst
 }
@@ -300,8 +357,8 @@ func (sa sumAggregator) Merge(src *uint64, dst *uint64) *uint64 {
 // files.
 func SumAnnotator(accumulate func(f *FileMetadata) (v uint64, cacheOK bool)) *Annotator[uint64] {
 	return &Annotator[uint64]{
-		Aggregator: sumAggregator{
-			accumulateFunc: accumulate,
+		Aggregator: SumAggregator{
+			AccumulateFunc: accumulate,
 		},
 	}
 }
diff --git a/metamorphic/options.go b/metamorphic/options.go
index 9fe771d5..b3538d98 100644
--- a/metamorphic/options.go
+++ b/metamorphic/options.go
@@ -57,7 +57,7 @@ func parseOptions(
 				return true
 			case "TestOptions.strictfs":
 				opts.strictFS = true
-				opts.Opts.FS = vfs.NewStrictMem()
+				opts.Opts.FS = vfs.NewCrashableMem()
 				return true
 			case "TestOptions.ingest_using_apply":
 				opts.ingestUsingApply = true
@@ -755,7 +755,7 @@ func RandomOptions(
 	testOpts.Threads = rng.Intn(runtime.GOMAXPROCS(0)) + 1
 	if testOpts.strictFS {
 		opts.DisableWAL = false
-		opts.FS = vfs.NewStrictMem()
+		opts.FS = vfs.NewCrashableMem()
 	} else if !testOpts.useDisk {
 		opts.FS = vfs.NewMem()
 	}
diff --git a/open.go b/open.go
index 46ecaa49..72de7a0c 100644
--- a/open.go
+++ b/open.go
@@ -408,6 +408,24 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
 	d.newIters = d.tableCache.newIters
 	d.tableNewRangeKeyIter = tableNewRangeKeyIter(d.newIters)
 
+	d.mu.annotators.totalSize = d.makeFileSizeAnnotator(func(f *manifest.FileMetadata) bool {
+		return true
+	})
+	d.mu.annotators.remoteSize = d.makeFileSizeAnnotator(func(f *manifest.FileMetadata) bool {
+		meta, err := d.objProvider.Lookup(fileTypeTable, f.FileBacking.DiskFileNum)
+		if err != nil {
+			return false
+		}
+		return meta.IsRemote()
+	})
+	d.mu.annotators.externalSize = d.makeFileSizeAnnotator(func(f *manifest.FileMetadata) bool {
+		meta, err := d.objProvider.Lookup(fileTypeTable, f.FileBacking.DiskFileNum)
+		if err != nil {
+			return false
+		}
+		return meta.IsRemote() && meta.Remote.CleanupMethod == objstorage.SharedNoCleanup
+	})
+
 	var previousOptionsFileNum base.DiskFileNum
 	var previousOptionsFilename string
 	for _, filename := range ls {
diff --git a/table_cache.go b/table_cache.go
index 849a842c..67bba2e4 100644
--- a/table_cache.go
+++ b/table_cache.go
@@ -210,27 +210,11 @@ func (c *tableCacheContainer) metrics() (CacheMetrics, FilterMetrics) {
 func (c *tableCacheContainer) estimateSize(
 	meta *fileMetadata, lower, upper []byte,
 ) (size uint64, err error) {
-	if meta.Virtual {
-		err = c.withVirtualReader(
-			meta.VirtualMeta(),
-			func(r sstable.VirtualReader) (err error) {
-				size, err = r.EstimateDiskUsage(lower, upper)
-				return err
-			},
-		)
-	} else {
-		err = c.withReader(
-			meta.PhysicalMeta(),
-			func(r *sstable.Reader) (err error) {
-				size, err = r.EstimateDiskUsage(lower, upper)
-				return err
-			},
-		)
-	}
-	if err != nil {
-		return 0, err
-	}
-	return size, nil
+	c.withCommonReader(meta, func(cr sstable.CommonReader) error {
+		size, err = cr.EstimateDiskUsage(lower, upper)
+		return err
+	})
+	return size, err
 }
 
 // createCommonReader creates a Reader for this file.
diff --git a/vfs/mem_fs.go b/vfs/mem_fs.go
index 1bb5fd8e..6b507059 100644
--- a/vfs/mem_fs.go
+++ b/vfs/mem_fs.go
@@ -8,6 +8,8 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"maps"
+	"math/rand"
 	"os"
 	"path"
 	"slices"
@@ -32,35 +34,34 @@ func NewMem() *MemFS {
 	}
 }
 
-// NewStrictMem returns a "strict" memory-backed FS implementation. The
-// behaviour is strict wrt needing a Sync() call on files or directories for the
-// state changes to be finalized. Any changes that are not finalized are visible
-// to reads until MemFS.ResetToSyncedState() is called, at which point they are
-// discarded and no longer visible. All open files must be closed before calling
-// ResetToSyncedState().
+// NewCrashableMem returns a memory-backed FS implementation that supports the
+// CrashClone() method. This method can be used to obtain a copy of the FS after
+// a simulated crash, where only data that was last synced is guaranteed to be
+// there (with no guarantees one way or the other about more recently written
+// data).
+//
+// Note: when CrashClone() is not necessary, NewMem() is faster and should be
+// preferred.
 //
 // Expected usage:
 //
-//	strictFS := NewStrictMem()
-//	db := Open(..., &Options{FS: strictFS})
-//	// Do and commit various operations.
-//	...
-//	// Prevent any more changes to finalized state.
-//	strictFS.SetIgnoreSyncs(true)
-//	// This will finish any ongoing background flushes, compactions but none of these writes will
-//	// be finalized since syncs are being ignored.
-//	db.Close()
-//	// Discard unsynced state.
-//	strictFS.ResetToSyncedState()
-//	// Allow changes to finalized state.
-//	strictFS.SetIgnoreSyncs(false)
-//	// Open the DB. This DB should have the same state as if the earlier strictFS operations and
-//	// db.Close() were not called.
-//	db := Open(..., &Options{FS: strictFS})
-func NewStrictMem() *MemFS {
+//		fs := NewCrashableMem()
+//		db := Open(..., &Options{FS: fs})
+//		// Do and commit various operations.
+//		...
+//		// Make a clone of the FS after a simulated crash.
+//	 crashedFS := fs.CrashClone(CrashCloneCfg{Probability: 50, RNG: rand.New(rand.NewSource(0))})
+//
+//		// This will finish any ongoing background flushes, compactions but none of these writes will
+//		// affect crashedFS.
+//		db.Close()
+//
+//		// Open the DB against the crash clone.
+//		db := Open(..., &Options{FS: crashedFS})
+func NewCrashableMem() *MemFS {
 	return &MemFS{
-		root:   newRootMemNode(),
-		strict: true,
+		root:      newRootMemNode(),
+		crashable: true,
 	}
 }
 
@@ -82,13 +83,16 @@ type MemFS struct {
 	mu   sync.Mutex
 	root *memNode
 
+	// cloneMu is used to block all modification operations while we clone the
+	// filesystem. Only used when crashable is true.
+	cloneMu sync.RWMutex
+
 	// lockFiles holds a map of open file locks. Presence in this map indicates
 	// a file lock is currently held. Keys are strings holding the path of the
 	// locked file. The stored value is untyped and  unused; only presence of
 	// the key within the map is significant.
 	lockedFiles sync.Map
-	strict      bool
-	ignoreSyncs bool
+	crashable   bool
 	// Windows has peculiar semantics with respect to hard links and deleting
 	// open files. In tests meant to exercise this behavior, this flag can be
 	// set to error if removing an open file.
@@ -116,33 +120,33 @@ func (y *MemFS) String() string {
 	return s.String()
 }
 
-// SetIgnoreSyncs sets the MemFS.ignoreSyncs field. See the usage comment with NewStrictMem() for
-// details.
-func (y *MemFS) SetIgnoreSyncs(ignoreSyncs bool) {
-	if !y.strict {
-		panic("SetIgnoreSyncs can only be used on a strict MemFS")
-	}
-	y.mu.Lock()
-	y.ignoreSyncs = ignoreSyncs
-	y.mu.Unlock()
+// CrashCloneCfg configures a CrashClone call. The zero value corresponds to the
+// crash clone containing exactly the data that was last synced.
+type CrashCloneCfg struct {
+	// UnsyncedDataPercent is the probability that a data block or directory entry
+	// that was not synced will be part of the clone. If 0, the clone will contain
+	// exactly the data that was last synced. If 100, the clone will be identical
+	// to the current filesystem.
+	UnsyncedDataPercent int
+	// RNG must be set if UnsyncedDataPercent > 0.
+	RNG *rand.Rand
 }
 
-// ResetToSyncedState discards state in the FS that is not synced. See the usage
-// comment with NewStrictMem() for details. All files must be closed before
-// calling this method.
-func (y *MemFS) ResetToSyncedState() {
-	if !y.strict {
-		panic("ResetToSyncedState can only be used on a strict MemFS")
+// CrashClone creates a new filesystem that reflects a possible state of this
+// filesystem after a crash at this moment. The new filesystem will contain all
+// data that was synced, and some fraction of the data that was not synced. The
+// latter is controlled by CrashCloneCfg.
+func (y *MemFS) CrashClone(cfg CrashCloneCfg) *MemFS {
+	if !y.crashable {
+		panic("not a strict MemFS")
 	}
-	y.mu.Lock()
-	y.root.descend("", func(path string, n *memNode) error {
-		if v := n.refs.Load(); v > 0 {
-			return errors.AssertionFailedf("file %s has %d open references; cannot reset to synced state", path, v)
-		}
-		return nil
-	})
-	y.root.resetToSyncedState()
-	y.mu.Unlock()
+	// Block all modification operations while we clone.
+	y.cloneMu.Lock()
+	defer y.cloneMu.Unlock()
+	newFS := &MemFS{crashable: true}
+	newFS.windowsSemantics = y.windowsSemantics
+	newFS.root = y.root.CrashClone(&cfg)
+	return newFS
 }
 
 // walk walks the directory tree for the fullname, calling f at each step. If
@@ -215,6 +219,10 @@ func (y *MemFS) walk(fullname string, f func(dir *memNode, frag string, final bo
 
 // Create implements FS.Create.
 func (y *MemFS) Create(fullname string, category DiskWriteCategory) (File, error) {
+	if y.crashable {
+		y.cloneMu.RLock()
+		defer y.cloneMu.RUnlock()
+	}
 	var ret *memFile
 	err := y.walk(fullname, func(dir *memNode, frag string, final bool) error {
 		if final {
@@ -242,6 +250,10 @@ func (y *MemFS) Create(fullname string, category DiskWriteCategory) (File, error
 
 // Link implements FS.Link.
 func (y *MemFS) Link(oldname, newname string) error {
+	if y.crashable {
+		y.cloneMu.RLock()
+		defer y.cloneMu.RUnlock()
+	}
 	var n *memNode
 	err := y.walk(oldname, func(dir *memNode, frag string, final bool) error {
 		if final {
@@ -268,6 +280,8 @@ func (y *MemFS) Link(oldname, newname string) error {
 			if frag == "" {
 				return errors.New("pebble/vfs: empty file name")
 			}
+			y.cloneMu.RLock()
+			defer y.cloneMu.RUnlock()
 			if _, ok := dir.children[frag]; ok {
 				return &os.LinkError{
 					Op:  "link",
@@ -344,6 +358,10 @@ func (y *MemFS) OpenDir(fullname string) (File, error) {
 
 // Remove implements FS.Remove.
 func (y *MemFS) Remove(fullname string) error {
+	if y.crashable {
+		y.cloneMu.RLock()
+		defer y.cloneMu.RUnlock()
+	}
 	return y.walk(fullname, func(dir *memNode, frag string, final bool) error {
 		if final {
 			if frag == "" {
@@ -373,6 +391,10 @@ func (y *MemFS) Remove(fullname string) error {
 
 // RemoveAll implements FS.RemoveAll.
 func (y *MemFS) RemoveAll(fullname string) error {
+	if y.crashable {
+		y.cloneMu.RLock()
+		defer y.cloneMu.RUnlock()
+	}
 	err := y.walk(fullname, func(dir *memNode, frag string, final bool) error {
 		if final {
 			if frag == "" {
@@ -396,6 +418,10 @@ func (y *MemFS) RemoveAll(fullname string) error {
 
 // Rename implements FS.Rename.
 func (y *MemFS) Rename(oldname, newname string) error {
+	if y.crashable {
+		y.cloneMu.RLock()
+		defer y.cloneMu.RUnlock()
+	}
 	var n *memNode
 	err := y.walk(oldname, func(dir *memNode, frag string, final bool) error {
 		if final {
@@ -430,6 +456,10 @@ func (y *MemFS) Rename(oldname, newname string) error {
 
 // ReuseForWrite implements FS.ReuseForWrite.
 func (y *MemFS) ReuseForWrite(oldname, newname string, category DiskWriteCategory) (File, error) {
+	if y.crashable {
+		y.cloneMu.RLock()
+		defer y.cloneMu.RUnlock()
+	}
 	if err := y.Rename(oldname, newname); err != nil {
 		return nil, err
 	}
@@ -448,6 +478,10 @@ func (y *MemFS) ReuseForWrite(oldname, newname string, category DiskWriteCategor
 
 // MkdirAll implements FS.MkdirAll.
 func (y *MemFS) MkdirAll(dirname string, perm os.FileMode) error {
+	if y.crashable {
+		y.cloneMu.RLock()
+		defer y.cloneMu.RUnlock()
+	}
 	return y.walk(dirname, func(dir *memNode, frag string, final bool) error {
 		if frag == "" {
 			if final {
@@ -476,6 +510,10 @@ func (y *MemFS) MkdirAll(dirname string, perm os.FileMode) error {
 
 // Lock implements FS.Lock.
 func (y *MemFS) Lock(fullname string) (io.Closer, error) {
+	if y.crashable {
+		y.cloneMu.RLock()
+		defer y.cloneMu.RUnlock()
+	}
 	// FS.Lock excludes other processes, but other processes cannot see this
 	// process' memory. However some uses (eg, Cockroach tests) may open and
 	// close the same MemFS-backed database multiple times. We want mutual
@@ -596,6 +634,13 @@ func newRootMemNode() *memNode {
 	}
 }
 
+func cloneChildren(f map[string]*memNode) map[string]*memNode {
+	if len(f) == 0 {
+		return make(map[string]*memNode)
+	}
+	return maps.Clone(f)
+}
+
 func (f *memNode) dump(w *bytes.Buffer, level int, name string) {
 	if f.isDir {
 		w.WriteString("          ")
@@ -626,29 +671,39 @@ func (f *memNode) dump(w *bytes.Buffer, level int, name string) {
 	}
 }
 
-func (f *memNode) descend(path string, fn func(string, *memNode) error) {
-	fn(path, f)
+func (f *memNode) CrashClone(cfg *CrashCloneCfg) *memNode {
+	newNode := &memNode{isDir: f.isDir}
 	if f.isDir {
-		for name, v := range f.children {
-			fn(path+sep+name, v)
-		}
-	}
-}
-
-func (f *memNode) resetToSyncedState() {
-	if f.isDir {
-		f.children = make(map[string]*memNode)
-		for k, v := range f.syncedChildren {
-			f.children[k] = v
+		newNode.children = cloneChildren(f.syncedChildren)
+		// Randomly include some non-synced children.
+		for name, child := range f.children {
+			if cfg.UnsyncedDataPercent > 0 && cfg.RNG.Intn(100) < cfg.UnsyncedDataPercent {
+				newNode.children[name] = child
+			}
 		}
-		for _, v := range f.children {
-			v.resetToSyncedState()
+		for name, child := range newNode.children {
+			newNode.children[name] = child.CrashClone(cfg)
 		}
+		newNode.syncedChildren = cloneChildren(newNode.children)
 	} else {
-		f.mu.Lock()
-		f.mu.data = slices.Clone(f.mu.syncedData)
-		f.mu.Unlock()
+		newNode.mu.data = slices.Clone(f.mu.syncedData)
+		newNode.mu.modTime = f.mu.modTime
+		// Randomly include some non-synced blocks.
+		const blockSize = 4096
+		for i := 0; i < len(f.mu.data); i += blockSize {
+			if cfg.UnsyncedDataPercent > 0 && cfg.RNG.Intn(100) < cfg.UnsyncedDataPercent {
+				block := f.mu.data[i:min(i+blockSize, len(f.mu.data))]
+				if grow := i + len(block) - len(newNode.mu.data); grow > 0 {
+					// Grow the file, leaving 0s for any unsynced blocks past the synced
+					// length.
+					newNode.mu.data = append(newNode.mu.data, make([]byte, grow)...)
+				}
+				copy(newNode.mu.data[i:], block)
+			}
+		}
+		newNode.mu.syncedData = slices.Clone(newNode.mu.data)
 	}
+	return newNode
 }
 
 // memFile is a reader or writer of a node's data. Implements File.
@@ -709,6 +764,10 @@ func (f *memFile) ReadAt(p []byte, off int64) (int, error) {
 }
 
 func (f *memFile) Write(p []byte) (int, error) {
+	if f.fs.crashable {
+		f.fs.cloneMu.RLock()
+		defer f.fs.cloneMu.RUnlock()
+	}
 	if !f.write {
 		return 0, errors.New("pebble/vfs: file was not created for writing")
 	}
@@ -742,6 +801,10 @@ func (f *memFile) Write(p []byte) (int, error) {
 }
 
 func (f *memFile) WriteAt(p []byte, ofs int64) (int, error) {
+	if f.fs.crashable {
+		f.fs.cloneMu.RLock()
+		defer f.fs.cloneMu.RUnlock()
+	}
 	if !f.write {
 		return 0, errors.New("pebble/vfs: file was not created for writing")
 	}
@@ -779,22 +842,18 @@ func (f *memFile) Stat() (FileInfo, error) {
 }
 
 func (f *memFile) Sync() error {
-	if f.fs == nil || !f.fs.strict {
+	if f.fs == nil || !f.fs.crashable {
 		return nil
 	}
+	f.fs.cloneMu.RLock()
+	defer f.fs.cloneMu.RUnlock()
 	f.fs.mu.Lock()
 	defer f.fs.mu.Unlock()
-	if f.fs.ignoreSyncs {
-		return nil
-	}
 	if f.n.isDir {
-		f.n.syncedChildren = make(map[string]*memNode)
-		for k, v := range f.n.children {
-			f.n.syncedChildren[k] = v
-		}
+		f.n.syncedChildren = cloneChildren(f.n.children)
 	} else {
 		f.n.mu.Lock()
-		f.n.mu.syncedData = slices.Clone(f.n.mu.data)
+		f.n.mu.syncedData = append(f.n.mu.syncedData[:0], f.n.mu.data...)
 		f.n.mu.Unlock()
 	}
 	return nil


Test patch:
diff --git a/checkpoint_test.go b/checkpoint_test.go
index 54e3127e..cf2d907e 100644
--- a/checkpoint_test.go
+++ b/checkpoint_test.go
@@ -327,7 +327,7 @@ func TestCheckpointCompaction(t *testing.T) {
 
 func TestCheckpointFlushWAL(t *testing.T) {
 	const checkpointPath = "checkpoints/checkpoint"
-	fs := vfs.NewStrictMem()
+	fs := vfs.NewCrashableMem()
 	opts := &Options{FS: fs, Logger: testLogger{t: t}}
 	key, value := []byte("key"), []byte("value")
 
@@ -345,7 +345,7 @@ func TestCheckpointFlushWAL(t *testing.T) {
 		err = d.Checkpoint(checkpointPath, WithFlushedWAL())
 		require.NoError(t, err)
 		require.NoError(t, d.Close())
-		fs.ResetToSyncedState()
+		fs = fs.CrashClone(vfs.CrashCloneCfg{UnsyncedDataPercent: 0})
 	}
 
 	// Check that the WAL has been flushed in the checkpoint.
diff --git a/error_test.go b/error_test.go
index c152c92c..11b8d3a5 100644
--- a/error_test.go
+++ b/error_test.go
@@ -352,12 +352,13 @@ L6:
 }
 
 func TestDBWALRotationCrash(t *testing.T) {
-	memfs := vfs.NewStrictMem()
+	memfs := vfs.NewCrashableMem()
 
+	var crashFS *vfs.MemFS
 	var index atomic.Int32
 	inj := errorfs.InjectorFunc(func(op errorfs.Op) error {
 		if op.Kind.ReadOrWrite() == errorfs.OpIsWrite && index.Add(-1) == -1 {
-			memfs.SetIgnoreSyncs(true)
+			crashFS = memfs.CrashClone(vfs.CrashCloneCfg{UnsyncedDataPercent: 0})
 		}
 		return nil
 	})
@@ -392,8 +393,7 @@ func TestDBWALRotationCrash(t *testing.T) {
 
 	fs := errorfs.Wrap(memfs, inj)
 	for k := int32(0); ; k++ {
-		// Run, simulating a crash by ignoring syncs after the k-th write
-		// operation after Open.
+		// Run, simulating a crash after the k-th write operation after Open.
 		index.Store(math.MaxInt32)
 		err := run(fs, k)
 		if !triggered() {
@@ -409,8 +409,7 @@ func TestDBWALRotationCrash(t *testing.T) {
 
 		// Reset the filesystem to its state right before the simulated
 		// "crash", restore syncs, and run again without crashing.
-		memfs.ResetToSyncedState()
-		memfs.SetIgnoreSyncs(false)
+		memfs = crashFS
 		index.Store(math.MaxInt32)
 		require.NoError(t, run(fs, k))
 	}
@@ -428,15 +427,19 @@ func TestDBCompactionCrash(t *testing.T) {
 	// crashIndex holds the value of k at which the crash is induced and is
 	// decremented by the errorfs on each write operation.
 	var crashIndex atomic.Int32
-	mkFS := func() (vfs.FS, *vfs.MemFS) {
-		memfs := vfs.NewStrictMem()
+	var crashFS *vfs.MemFS
+	crashRNG := rand.New(rand.NewSource(seed))
+	mkFS := func() vfs.FS {
+		memfs := vfs.NewCrashableMem()
 		inj := errorfs.InjectorFunc(func(op errorfs.Op) error {
 			if op.Kind.ReadOrWrite() == errorfs.OpIsWrite && crashIndex.Add(-1) == -1 {
-				memfs.SetIgnoreSyncs(true)
+				// Allow an arbitrary subset of non-synced state to survive beyond the
+				// crash point.
+				crashFS = memfs.CrashClone(vfs.CrashCloneCfg{UnsyncedDataPercent: 10, RNG: crashRNG})
 			}
 			return nil
 		})
-		return errorfs.Wrap(memfs, inj), memfs
+		return errorfs.Wrap(memfs, inj)
 	}
 	triggered := func() bool { return crashIndex.Load() < 0 }
 
@@ -523,8 +526,7 @@ func TestDBCompactionCrash(t *testing.T) {
 			// Run, simulating a crash by ignoring syncs after the k-th write
 			// operation after Open.
 			crashIndex.Store(math.MaxInt32)
-			fs, memfs := mkFS()
-			i, err := run(t, fs, k, seed)
+			i, err := run(t, mkFS(), k, seed)
 			if !triggered() {
 				// Stop when we reach a value of k greater than the number of
 				// write operations performed during `run`.
@@ -540,13 +542,8 @@ func TestDBCompactionCrash(t *testing.T) {
 			// Reset the filesystem to its state right before the simulated
 			// "crash", restore syncs and run again without crashing. No errors
 			// should be encountered.
-			//
-			// TODO(jackson): Allow an arbitrary subset of synced state to
-			// survive beyond the crash point.
-			memfs.ResetToSyncedState()
-			memfs.SetIgnoreSyncs(false)
 			crashIndex.Store(math.MaxInt32)
-			_, err = run(t, fs, math.MaxInt32, seed)
+			_, err = run(t, crashFS, math.MaxInt32, seed)
 			require.False(t, triggered())
 			// TODO(jackson): Add assertions on the database keys.
 			require.NoError(t, err)
diff --git a/format_major_version_test.go b/format_major_version_test.go
index b2e7128f..3cf01976 100644
--- a/format_major_version_test.go
+++ b/format_major_version_test.go
@@ -102,7 +102,7 @@ func testBasicDB(d *DB) error {
 func TestFormatMajorVersions(t *testing.T) {
 	for vers := FormatMinSupported; vers <= FormatNewest; vers++ {
 		t.Run(fmt.Sprintf("vers=%03d", vers), func(t *testing.T) {
-			fs := vfs.NewStrictMem()
+			fs := vfs.NewCrashableMem()
 			opts := (&Options{
 				FS:                 fs,
 				FormatMajorVersion: vers,
@@ -126,18 +126,15 @@ func TestFormatMajorVersions(t *testing.T) {
 			t.Run("upgrade-at-open", func(t *testing.T) {
 				for upgradeVers := vers + 1; upgradeVers <= FormatNewest; upgradeVers++ {
 					t.Run(fmt.Sprintf("upgrade-vers=%03d", upgradeVers), func(t *testing.T) {
-						// We use vfs.MemFS's option to ignore syncs so
-						// that we can perform an upgrade on the current
-						// database state in fs, and revert it when this
-						// subtest is complete.
-						fs.SetIgnoreSyncs(true)
-						defer fs.ResetToSyncedState()
+						// We use vfs.MemFS's CrashClone to perform an upgrade without
+						// affecting the original filesystem.
+						opts := opts.Clone()
+						opts.FS = fs.CrashClone(vfs.CrashCloneCfg{UnsyncedDataPercent: 0})
 
 						// Re-open the database, passing a higher format
 						// major version in the Options to automatically
 						// ratchet the format major version. Ensure some
 						// basic operations pass.
-						opts := opts.Clone()
 						opts.FormatMajorVersion = upgradeVers
 						d, err = Open("", opts)
 						require.NoError(t, err)
@@ -162,12 +159,10 @@ func TestFormatMajorVersions(t *testing.T) {
 						// options.
 						require.Equal(t, vers, opts.FormatMajorVersion)
 
-						// We use vfs.MemFS's option to ignore syncs so
-						// that we can perform an upgrade on the current
-						// database state in fs, and revert it when this
-						// subtest is complete.
-						fs.SetIgnoreSyncs(true)
-						defer fs.ResetToSyncedState()
+						// We use vfs.MemFS's CrashClone to perform an upgrade without
+						// affecting the original filesystem.
+						opts := opts.Clone()
+						opts.FS = fs.CrashClone(vfs.CrashCloneCfg{UnsyncedDataPercent: 0})
 
 						// Re-open the database, still at the current format
 						// major version. Perform some basic operations,
diff --git a/ingest_test.go b/ingest_test.go
index 200365ba..8752a04a 100644
--- a/ingest_test.go
+++ b/ingest_test.go
@@ -422,7 +422,8 @@ func TestIngestLinkFallback(t *testing.T) {
 func TestOverlappingIngestedSSTs(t *testing.T) {
 	dir := ""
 	var (
-		mem        vfs.FS
+		mem        *vfs.MemFS
+		crashClone *vfs.MemFS
 		d          *DB
 		opts       *Options
 		closed     = false
@@ -446,7 +447,7 @@ func TestOverlappingIngestedSSTs(t *testing.T) {
 		blockFlush = false
 
 		if strictMem {
-			mem = vfs.NewStrictMem()
+			mem = vfs.NewCrashableMem()
 		} else {
 			mem = vfs.NewMem()
 		}
@@ -513,16 +514,13 @@ func TestOverlappingIngestedSSTs(t *testing.T) {
 			reset(td.HasArg("strictMem"))
 			return ""
 
-		case "ignoreSyncs":
-			var ignoreSyncs bool
-			if len(td.CmdArgs) == 1 && td.CmdArgs[0].String() == "true" {
-				ignoreSyncs = true
-			}
-			mem.(*vfs.MemFS).SetIgnoreSyncs(ignoreSyncs)
+		case "crash-clone":
+			crashClone = mem.CrashClone(vfs.CrashCloneCfg{UnsyncedDataPercent: 0})
 			return ""
 
-		case "resetToSynced":
-			mem.(*vfs.MemFS).ResetToSyncedState()
+		case "reset-to-crash-clone":
+			mem = crashClone
+			crashClone = nil
 			files, err := mem.List(dir)
 			sort.Strings(files)
 			require.NoError(t, err)
diff --git a/iterator_test.go b/iterator_test.go
index 7d5bc63d..0b378641 100644
--- a/iterator_test.go
+++ b/iterator_test.go
@@ -2107,7 +2107,7 @@ func TestRangeKeyMaskingRandomized(t *testing.T) {
 	maxProcs := runtime.GOMAXPROCS(0)
 
 	opts1 := &Options{
-		FS:                       vfs.NewStrictMem(),
+		FS:                       vfs.NewCrashableMem(),
 		Comparer:                 testkeys.Comparer,
 		FormatMajorVersion:       FormatNewest,
 		MaxConcurrentCompactions: func() int { return maxProcs/2 + 1 },
@@ -2120,7 +2120,7 @@ func TestRangeKeyMaskingRandomized(t *testing.T) {
 	require.NoError(t, err)
 
 	opts2 := &Options{
-		FS:                       vfs.NewStrictMem(),
+		FS:                       vfs.NewCrashableMem(),
 		Comparer:                 testkeys.Comparer,
 		FormatMajorVersion:       FormatNewest,
 		MaxConcurrentCompactions: func() int { return maxProcs/2 + 1 },
@@ -2245,7 +2245,7 @@ func BenchmarkIterator_RangeKeyMasking(b *testing.B) {
 	keyBuf := make([]byte, prefixLen+testkeys.MaxSuffixLen)
 	valBuf := make([]byte, valueSize)
 
-	mem := vfs.NewStrictMem()
+	mem := vfs.NewMem()
 	maxProcs := runtime.GOMAXPROCS(0)
 	opts := &Options{
 		FS:                       mem,
@@ -2285,14 +2285,17 @@ func BenchmarkIterator_RangeKeyMasking(b *testing.B) {
 	d.mu.Unlock()
 	b.Log(d.Metrics().String())
 	require.NoError(b, d.Close())
-	// Set ignore syncs to true so that each subbenchmark may mutate state and
-	// then revert back to the original state.
-	mem.SetIgnoreSyncs(true)
 
 	// TODO(jackson): Benchmark lazy-combined iteration versus not.
 	// TODO(jackson): Benchmark seeks.
 	for _, rkSuffix := range []string{"@10", "@50", "@75", "@100"} {
 		b.Run(fmt.Sprintf("range-keys-suffixes=%s", rkSuffix), func(b *testing.B) {
+			// Clone the filesystem so that each subbenchmark may mutate state.
+			opts := opts.Clone()
+			opts.FS = vfs.NewMem()
+			ok, err := vfs.Clone(mem, opts.FS, "", "")
+			require.NoError(b, err)
+			require.True(b, ok)
 			d, err := Open("", opts)
 			require.NoError(b, err)
 			require.NoError(b, d.RangeKeySet([]byte("b"), []byte("e"), []byte(rkSuffix), nil, nil))
@@ -2367,10 +2370,8 @@ func BenchmarkIterator_RangeKeyMasking(b *testing.B) {
 			// range keys we wrote.
 			b.StopTimer()
 			require.NoError(b, d.Close())
-			mem.ResetToSyncedState()
 		})
 	}
-
 }
 
 func BenchmarkIteratorScan(b *testing.B) {
diff --git a/metamorphic/test.go b/metamorphic/test.go
index 48ab870a..add8f3f7 100644
--- a/metamorphic/test.go
+++ b/metamorphic/test.go
@@ -286,9 +286,8 @@ func (t *Test) restartDB(dbID objID) error {
 		return nil
 	}
 	t.opts.Cache.Ref()
-
-	memFS := vfs.Root(t.opts.FS).(*vfs.MemFS)
-	memFS.SetIgnoreSyncs(true)
+	fs := vfs.Root(t.opts.FS).(*vfs.MemFS)
+	crashFS := fs.CrashClone(vfs.CrashCloneCfg{UnsyncedDataPercent: 0})
 	if err := db.Close(); err != nil {
 		return err
 	}
@@ -300,9 +299,11 @@ func (t *Test) restartDB(dbID objID) error {
 			return err
 		}
 	}
-
-	memFS.ResetToSyncedState()
-	memFS.SetIgnoreSyncs(false)
+	t.opts.FS = crashFS
+	t.opts.WithFSDefaults()
+	if t.opts.WALFailover != nil {
+		t.opts.WALFailover.Secondary.FS = t.opts.FS
+	}
 
 	// TODO(jackson): Audit errorRate and ensure custom options' hooks semantics
 	// are well defined within the context of retries.
diff --git a/objstorage/objstorageprovider/provider_test.go b/objstorage/objstorageprovider/provider_test.go
index 67e56870..0b70414e 100644
--- a/objstorage/objstorageprovider/provider_test.go
+++ b/objstorage/objstorageprovider/provider_test.go
@@ -583,7 +583,7 @@ func TestParallelSync(t *testing.T) {
 			name = "shared"
 		}
 		t.Run(name, func(t *testing.T) {
-			fs := vfs.NewStrictMem()
+			fs := vfs.NewCrashableMem()
 			st := DefaultSettings(fs, "")
 			st.Remote.StorageFactory = remote.MakeSimpleFactory(map[remote.Locator]remote.Storage{
 				"": remote.NewInMem(),
@@ -660,6 +660,7 @@ func TestParallelSync(t *testing.T) {
 			}
 			mustExist := make(map[base.DiskFileNum]struct{})
 			// "Crash" at a random time.
+			var crashFS *vfs.MemFS
 			time.AfterFunc(time.Duration(rand.Int63n(int64(10*time.Millisecond))), func() {
 				defer wg.Done()
 				if shared {
@@ -673,7 +674,7 @@ func TestParallelSync(t *testing.T) {
 				for n := range mustExistMu.m {
 					mustExist[n] = struct{}{}
 				}
-				fs.SetIgnoreSyncs(true)
+				crashFS = fs.CrashClone(vfs.CrashCloneCfg{})
 				mustExistMu.Unlock()
 				stop.Store(true)
 			})
@@ -682,8 +683,10 @@ func TestParallelSync(t *testing.T) {
 			// Now close the provider, reset the filesystem, and check that all files
 			// we expect to exist are there.
 			require.NoError(t, p.Close())
-			fs.ResetToSyncedState()
 
+			if !shared {
+				st.FS = crashFS
+			}
 			p, err = Open(st)
 			require.NoError(t, err)
 			// Check that all objects exist and can be opened.
diff --git a/open_test.go b/open_test.go
index 61763b2c..0b2e0ba2 100644
--- a/open_test.go
+++ b/open_test.go
@@ -928,7 +928,7 @@ func TestTwoWALReplayCorrupt(t *testing.T) {
 // new WAL is created, the current manifest's MinUnflushedLogNum must be
 // higher than the previous WAL.
 func TestCrashOpenCrashAfterWALCreation(t *testing.T) {
-	fs := vfs.NewStrictMem()
+	fs := vfs.NewCrashableMem()
 
 	getLogs := func() (logs []string) {
 		ls, err := fs.List("")
@@ -941,19 +941,17 @@ func TestCrashOpenCrashAfterWALCreation(t *testing.T) {
 		return logs
 	}
 
-	{
-		d, err := Open("", testingRandomized(t, &Options{FS: fs}))
-		require.NoError(t, err)
-		require.NoError(t, d.Set([]byte("abc"), nil, Sync))
+	d, err := Open("", testingRandomized(t, &Options{FS: fs}))
+	require.NoError(t, err)
+	require.NoError(t, d.Set([]byte("abc"), nil, Sync))
 
-		// Ignore syncs during close to simulate a crash. This will leave the WAL
-		// without an EOF trailer. It won't be an 'unclean tail' yet since the
-		// log file was not recycled, but we'll fix that down below.
-		fs.SetIgnoreSyncs(true)
-		require.NoError(t, d.Close())
-		fs.ResetToSyncedState()
-		fs.SetIgnoreSyncs(false)
-	}
+	// simulate a crash. This will leave the WAL
+	// without an EOF trailer. It won't be an 'unclean tail' yet since the
+	// log file was not recycled, but we'll fix that down below.
+	crashFS := fs.CrashClone(vfs.CrashCloneCfg{UnsyncedDataPercent: 0})
+	require.NoError(t, d.Close())
+
+	fs = crashFS
 
 	// There should be one WAL.
 	logs := getLogs()
@@ -988,12 +986,15 @@ func TestCrashOpenCrashAfterWALCreation(t *testing.T) {
 	// filesystem with an errorfs that will turn off syncs after a new .log
 	// file is created and after a subsequent directory sync occurs. This
 	// simulates a crash after the new log file is created and synced.
+	crashFS = nil
 	{
 		var walCreated, dirSynced atomic.Bool
 		d, err := Open("", &Options{
 			FS: errorfs.Wrap(fs, errorfs.InjectorFunc(func(op errorfs.Op) error {
 				if dirSynced.Load() {
-					fs.SetIgnoreSyncs(true)
+					if crashFS == nil {
+						crashFS = fs.CrashClone(vfs.CrashCloneCfg{UnsyncedDataPercent: 0})
+					}
 				}
 				if op.Kind == errorfs.OpCreate && filepath.Ext(op.Path) == ".log" {
 					walCreated.Store(true)
@@ -1012,15 +1013,15 @@ func TestCrashOpenCrashAfterWALCreation(t *testing.T) {
 		require.NoError(t, d.Close())
 	}
 
-	fs.ResetToSyncedState()
-	fs.SetIgnoreSyncs(false)
+	require.NotNil(t, crashFS)
+	fs = crashFS
 
 	if n := len(getLogs()); n != 2 {
 		t.Fatalf("expected two logs, found %d\n", n)
 	}
 
 	// Finally, open the database with syncs enabled.
-	d, err := Open("", testingRandomized(t, &Options{FS: fs}))
+	d, err = Open("", testingRandomized(t, &Options{FS: fs}))
 	require.NoError(t, err)
 	require.NoError(t, d.Close())
 }
diff --git a/testdata/flushable_ingest b/testdata/flushable_ingest
index 31bb46c6..c5806832 100644
--- a/testdata/flushable_ingest
+++ b/testdata/flushable_ingest
@@ -551,7 +551,7 @@ b
 a:2
 b:2
 
-ignoreSyncs true
+crash-clone
 ----
 
 lsm
@@ -592,7 +592,7 @@ close
 
 # At this point, the changes to the manifest should be lost. Note that 7.sst
 # is gone because that file was never synced.
-resetToSynced
+reset-to-crash-clone
 ----
 000002.log
 000004.sst
@@ -606,9 +606,6 @@ ext1
 marker.format-version.000004.017
 marker.manifest.000001.MANIFEST-000001
 
-ignoreSyncs false
-----
-
 open
 ----
 
diff --git a/vfs/atomicfs/marker_test.go b/vfs/atomicfs/marker_test.go
index 2ffba22c..373103e3 100644
--- a/vfs/atomicfs/marker_test.go
+++ b/vfs/atomicfs/marker_test.go
@@ -166,7 +166,7 @@ func TestMarker(t *testing.T) {
 
 func TestMarker_StrictSync(t *testing.T) {
 	// Use an in-memory FS that strictly enforces syncs.
-	mem := vfs.NewStrictMem()
+	mem := vfs.NewCrashableMem()
 	syncDir := func(dir string) {
 		fdir, err := mem.OpenDir(dir)
 		require.NoError(t, err)
@@ -184,16 +184,16 @@ func TestMarker_StrictSync(t *testing.T) {
 
 	// Discard any unsynced writes to make sure we set up the test
 	// preconditions correctly.
-	mem.ResetToSyncedState()
-	m, v, err = LocateMarker(mem, "foo", "bar")
+	crashFS := mem.CrashClone(vfs.CrashCloneCfg{UnsyncedDataPercent: 0})
+	m, v, err = LocateMarker(crashFS, "foo", "bar")
 	require.NoError(t, err)
 	require.Equal(t, "hello", v)
 	require.NoError(t, m.Move("hello-world"))
 	require.NoError(t, m.Close())
 
 	// Discard any unsynced writes.
-	mem.ResetToSyncedState()
-	m, v, err = LocateMarker(mem, "foo", "bar")
+	crashFS = crashFS.CrashClone(vfs.CrashCloneCfg{UnsyncedDataPercent: 0})
+	m, v, err = LocateMarker(crashFS, "foo", "bar")
 	require.NoError(t, err)
 	require.Equal(t, "hello-world", v)
 	require.NoError(t, m.Close())
diff --git a/vfs/mem_fs_test.go b/vfs/mem_fs_test.go
index 61f6573a..60cf1492 100644
--- a/vfs/mem_fs_test.go
+++ b/vfs/mem_fs_test.go
@@ -7,6 +7,7 @@ package vfs
 import (
 	"fmt"
 	"io"
+	"math/rand"
 	"sort"
 	"strconv"
 	"strings"
@@ -17,7 +18,9 @@ import (
 )
 
 func runMemFSDataDriven(t *testing.T, path string, fs *MemFS) {
+	fsMap := map[string]*MemFS{"initial": fs}
 	var f File
+	rng := rand.New(rand.NewSource(0))
 	datadriven.RunTest(t, path, func(t *testing.T, td *datadriven.TestData) string {
 		var err error
 		switch td.Cmd {
@@ -39,12 +42,17 @@ func runMemFSDataDriven(t *testing.T, path string, fs *MemFS) {
 			err = fs.Rename(td.CmdArgs[0].String(), td.CmdArgs[1].String())
 		case "reuse-for-write":
 			f, err = fs.ReuseForWrite(td.CmdArgs[0].String(), td.CmdArgs[1].String(), WriteCategoryUnspecified)
-		case "reset-to-synced":
-			fs.ResetToSyncedState()
-		case "ignore-syncs":
-			fs.SetIgnoreSyncs(true)
-		case "stop-ignoring-syncs":
-			fs.SetIgnoreSyncs(false)
+		case "crash-clone":
+			p, _ := strconv.Atoi(td.CmdArgs[0].String())
+			fsName := td.CmdArgs[1].String()
+			newFs := fs.CrashClone(CrashCloneCfg{UnsyncedDataPercent: p, RNG: rng})
+			fsMap[fsName] = newFs
+		case "switch-fs":
+			fsName := td.CmdArgs[0].String()
+			fs = fsMap[fsName]
+			if fs == nil {
+				td.Fatalf(t, "no fs %q", fsName)
+			}
 		case "f.write":
 			_, err = f.Write([]byte(strings.TrimSpace(td.Input)))
 		case "f.sync":
@@ -116,8 +124,8 @@ func TestMemFSList(t *testing.T) {
 	runMemFSDataDriven(t, "testdata/memfs_list", NewMem())
 }
 
-func TestMemFSStrict(t *testing.T) {
-	runMemFSDataDriven(t, "testdata/memfs_strict", NewStrictMem())
+func TestMemFSCrashable(t *testing.T) {
+	runMemFSDataDriven(t, "testdata/memfs_crashable", NewCrashableMem())
 }
 
 func TestMemFile(t *testing.T) {
diff --git a/vfs/testdata/memfs_strict b/vfs/testdata/memfs_crashable
similarity index 81%
rename from vfs/testdata/memfs_strict
rename to vfs/testdata/memfs_crashable
index 03f9e885..18247707 100644
--- a/vfs/testdata/memfs_strict
+++ b/vfs/testdata/memfs_crashable
@@ -5,13 +5,19 @@ create /foo
 open /foo
 ----
 
-reset-to-synced
+crash-clone 0 fs0
+----
+
+switch-fs fs0
 ----
 
 open /foo
 ----
 error: open /foo: file does not exist
 
+switch-fs initial
+----
+
 # Create directory and a file in it and write and read from it.
 mkdirall /bar
 ----
@@ -45,7 +51,10 @@ open /bar
 ----
 
 # Resetting causes both the directory and file to disappear.
-reset-to-synced
+crash-clone 0 fs1
+----
+
+switch-fs fs1
 ----
 
 open-dir /bar
@@ -56,6 +65,9 @@ open /bar/y
 ----
 error: open bar/y: file does not exist
 
+switch-fs initial
+----
+
 # Create the directory and file again. Link the file to another file in the same
 # dir, and to a file in the root dir. Sync the root dir. After reset, the
 # created dir and the file in the root dir are the only ones visible.
@@ -89,7 +101,10 @@ f.sync
 f.close
 ----
 
-reset-to-synced
+crash-clone 0 fs2
+----
+
+switch-fs fs2
 ----
 
 open-dir /bar
@@ -106,6 +121,9 @@ error: open /bar/z: file does not exist
 open /z
 ----
 
+switch-fs initial
+----
+
 # Create the file in the directory again and this time sync /bar directory. The
 # file is preserved after reset.
 create /bar/y
@@ -120,7 +138,10 @@ f.sync
 f.close
 ----
 
-reset-to-synced
+crash-clone 0 fs3
+----
+
+switch-fs fs3
 ----
 
 open-dir /bar
@@ -129,6 +150,9 @@ open-dir /bar
 open /bar/y
 ----
 
+switch-fs initial
+----
+
 # Unsynced data in the file is lost on reset.
 create /bar/y
 ----
@@ -156,7 +180,10 @@ f.sync
 f.close
 ----
 
-reset-to-synced
+crash-clone 0 fs4
+----
+
+switch-fs fs4
 ----
 
 open /bar/y
@@ -173,8 +200,11 @@ error: EOF
 f.close
 ----
 
-# reuse-for-write works correctly in strict mode in that unsynced data does not overwrite
-# previous contents when a reset happens.
+switch-fs initial
+----
+
+# reuse-for-write works correctly in that unsynced data does not overwrite
+# previous contents when a crash happens.
 create /z
 ----
 
@@ -215,7 +245,10 @@ f.sync
 f.close
 ----
 
-reset-to-synced
+crash-clone 0 fs5
+----
+
+switch-fs fs5
 ----
 
 open /y
@@ -227,56 +260,3 @@ xbcdefgh
 
 f.close
 ----
-
-# Ignore syncs.
-create /z
-----
-
-f.write
-a
-----
-
-f.sync
-----
-
-ignore-syncs
-----
-
-f.write
-b
-----
-
-f.sync
-----
-
-f.close
-----
-
-stop-ignoring-syncs
-----
-
-open-dir /
-----
-
-f.sync
-----
-
-f.close
-----
-
-reset-to-synced
-----
-
-open /z
-----
-
-f.read 1
-----
-a
-
-f.read 1
-----
-error: EOF
-
-f.close
-----
diff --git a/wal/failover_writer_test.go b/wal/failover_writer_test.go
index 45c99d58..3c859525 100644
--- a/wal/failover_writer_test.go
+++ b/wal/failover_writer_test.go
@@ -38,7 +38,7 @@ const (
 
 func TestFailoverWriter(t *testing.T) {
 	datadriven.Walk(t, "testdata/failover_writer", func(t *testing.T, path string) {
-		memFS := vfs.NewStrictMem()
+		memFS := vfs.NewCrashableMem()
 		dirs := [numDirIndices]dirAndFileHandle{
 			{Dir: Dir{Dirname: "pri"}},
 			{Dir: Dir{Dirname: "sec"}},
@@ -69,7 +69,9 @@ func TestFailoverWriter(t *testing.T) {
 		dirIndex := 0
 
 		printLogFiles := func(b *strings.Builder, num NumWAL) {
-			memFS.ResetToSyncedState()
+			memFS = memFS.CrashClone(vfs.CrashCloneCfg{UnsyncedDataPercent: 0})
+			setDirsFunc(t, memFS, &dirs)
+			setDirsFunc(t, memFS, &testDirs)
 			type filenameAndFS struct {
 				name string
 				fs   vfs.FS
@@ -615,7 +617,7 @@ func TestConcurrentWritersWithManyRecords(t *testing.T) {
 		}
 	}
 	const numLogWriters = 4
-	memFS := vfs.NewStrictMem()
+	memFS := vfs.NewCrashableMem()
 	dirs := [numDirIndices]dirAndFileHandle{{Dir: Dir{Dirname: "pri"}}, {Dir: Dir{Dirname: "sec"}}}
 	for _, dir := range dirs {
 		require.NoError(t, memFS.MkdirAll(dir.Dirname, 0755))
diff --git a/wal/reader_test.go b/wal/reader_test.go
index cb252c90..8f202ef0 100644
--- a/wal/reader_test.go
+++ b/wal/reader_test.go
@@ -97,7 +97,7 @@ func TestList(t *testing.T) {
 // TestReader tests the virtual WAL reader that merges across multiple physical
 // log files.
 func TestReader(t *testing.T) {
-	fs := vfs.NewStrictMem()
+	fs := vfs.NewCrashableMem()
 	rng := rand.New(rand.NewSource(1))
 	var buf bytes.Buffer
 	datadriven.RunTest(t, "testdata/reader", func(t *testing.T, td *datadriven.TestData) string {
@@ -190,10 +190,9 @@ func TestReader(t *testing.T) {
 				}
 			}
 			if td.HasArg("close-unclean") {
-				fs.SetIgnoreSyncs(true)
+				crashFS := fs.CrashClone(vfs.CrashCloneCfg{UnsyncedDataPercent: 0})
 				require.NoError(t, w.Close())
-				fs.ResetToSyncedState()
-				fs.SetIgnoreSyncs(false)
+				fs = crashFS
 			} else {
 				require.NoError(t, w.Close())
 			}