Repository: cockroachdb/pebble (go)

Issue:
db: add Excise API
Add a new API that allows excising a span of the database without performing an accompanying ingestion. This is useful for recovering from an instance where externally linked sstables no longer exist or are corrupt.

Jira issue: PEBBLE-329

Pull request description:
db: add DB.Excise
Add an Excise method to the DB allowing an excise without an accompanying ingestion. This will be used in CockroachDB to recover from corruption by excising out corrupted sstables.

For now, this implementation uses DB.ingest. I think a future refactor should be able to clean these code paths up and potentially refactor excise to be independent of the ingest codepath. Today, it's necessary to reuse the logic to perform excises as 'flushable ingests.'

Closes #4286.

Patch:
diff --git a/db.go b/db.go
index 6f260ae7..4a47f7c3 100644
--- a/db.go
+++ b/db.go
@@ -1619,11 +1619,21 @@ func (d *DB) NewIterWithContext(ctx context.Context, o *IterOptions) (*Iterator,
 // will not prevent memtables from being released or sstables from being
 // deleted. Instead, a snapshot prevents deletion of sequence numbers
 // referenced by the snapshot.
+//
+// There exists one violation of a Snapshot's point-in-time guarantee: An excise
+// (see DB.Excise and DB.IngestAndExcise) that occurs after the snapshot's
+// creation will be observed by iterators created from the snapshot after the
+// excise. See NewEventuallyFileOnlySnapshot for a variant of NewSnapshot that
+// provides a full point-in-time guarantee.
 func (d *DB) NewSnapshot() *Snapshot {
+	// TODO(jackson): Consider removal of regular, non-eventually-file-only
+	// snapshots given they no longer provide a true point-in-time snapshot of
+	// the database due to excises. If we had a mechanism to construct a maximal
+	// key range, we could implement NewSnapshot in terms of
+	// NewEventuallyFileOnlySnapshot and provide a true point-in-time guarantee.
 	if err := d.closed.Load(); err != nil {
 		panic(err)
 	}
-
 	d.mu.Lock()
 	s := &Snapshot{
 		db:     d,
diff --git a/excise.go b/excise.go
index 02c58f3f..e249c1fd 100644
--- a/excise.go
+++ b/excise.go
@@ -8,11 +8,42 @@ import (
 	"context"
 	"slices"
 
+	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/manifest"
 )
 
+// Excise atomically deletes all data overlapping with the provided span. All
+// data overlapping with the span is removed, including from open snapshots.
+// Only currently-open iterators will still observe the removed data (because an
+// open iterator pins all memtables and sstables in its view of the LSM until
+// it's closed). Excise may initiate a flush if there exists unflushed data
+// overlapping the excise span.
+func (d *DB) Excise(ctx context.Context, span KeyRange) error {
+	if err := d.closed.Load(); err != nil {
+		panic(err)
+	}
+	if d.opts.ReadOnly {
+		return ErrReadOnly
+	}
+	// Excise is only supported on prefix keys.
+	if d.opts.Comparer.Split(span.Start) != len(span.Start) {
+		return errors.New("Excise called with suffixed start key")
+	}
+	if d.opts.Comparer.Split(span.End) != len(span.End) {
+		return errors.New("Excise called with suffixed end key")
+	}
+	if v := d.FormatMajorVersion(); v < FormatVirtualSSTables {
+		return errors.Newf(
+			"store has format major version %d; Excise requires at least %d",
+			v, FormatVirtualSSTables,
+		)
+	}
+	_, err := d.ingest(ctx, nil, nil, span, nil)
+	return err
+}
+
 // excise updates ve to include a replacement of the file m with new virtual
 // sstables that exclude exciseSpan, returning a slice of newly-created files if
 // any. If the entirety of m is deleted by exciseSpan, no new sstables are added
diff --git a/file_cache.go b/file_cache.go
index 65db546b..2c1c1496 100644
--- a/file_cache.go
+++ b/file_cache.go
@@ -107,9 +107,12 @@ type fileCacheHandle struct {
 	// This struct is only populated in race builds.
 	raceMu struct {
 		sync.Mutex
-		// iters maps sstable.Iterator objects to the stack trace recorded at
-		// creation time.
-		iters map[any][]byte
+		// nextRefID is the next ID to allocate for a new reference.
+		nextRefID uint64
+		// openRefs maps reference IDs to the stack trace recorded at creation
+		// time. It's used to track which call paths leaked open references to
+		// files.
+		openRefs map[uint64][]byte
 	}
 }
 
@@ -134,7 +137,7 @@ func (c *FileCache) newHandle(
 	t.readerOpts = readerOpts
 	t.readerOpts.FilterMetricsTracker = &sstable.FilterMetricsTracker{}
 	if invariants.RaceEnabled {
-		t.raceMu.iters = make(map[any][]byte)
+		t.raceMu.openRefs = make(map[uint64][]byte)
 	}
 	return t
 }
@@ -151,7 +154,7 @@ func (h *fileCacheHandle) Close() error {
 			err = errors.Errorf("leaked iterators: %d", errors.Safe(v))
 		} else {
 			var buf bytes.Buffer
-			for _, stack := range h.raceMu.iters {
+			for _, stack := range h.raceMu.openRefs {
 				fmt.Fprintf(&buf, "%s\n", stack)
 			}
 			err = errors.Errorf("leaked iterators: %d\n%s", errors.Safe(v), buf.String())
@@ -348,12 +351,7 @@ func NewFileCache(numShards int, size int) *FileCache {
 		v := vRef.Value()
 		handle := key.handle
 		v.readerProvider.init(c, key)
-		v.closeHook = func(iterator any) {
-			if invariants.RaceEnabled {
-				handle.raceMu.Lock()
-				delete(handle.raceMu.iters, iterator)
-				handle.raceMu.Unlock()
-			}
+		v.closeHook = func() {
 			// closeHook is called when an iterator is closed; the initialization of
 			// an iterator with this value will happen after a FindOrCreate() call
 			// with returns the same vRef.
@@ -572,17 +570,35 @@ func (h *fileCacheHandle) newPointIter(
 	if err != nil {
 		return nil, err
 	}
-	// NB: v.closeHook takes responsibility for calling unrefValue(v) here. Take
-	// care to avoid introducing an allocation here by adding a closure.
-	iter.SetCloseHook(v.closeHook)
-	handle.iterCount.Add(1)
+	// NB: closeHook (v.closeHook) takes responsibility for calling
+	// unrefValue(v) here. Take care to avoid introducing an allocation here by
+	// adding a closure.
+	closeHook := h.addReference(v)
+	iter.SetCloseHook(closeHook)
+	return iter, nil
+}
+
+func (h *fileCacheHandle) addReference(v *fileCacheValue) (closeHook func()) {
+	h.iterCount.Add(1)
+	closeHook = v.closeHook
 	if invariants.RaceEnabled {
 		stack := debug.Stack()
 		h.raceMu.Lock()
-		h.raceMu.iters[iter] = stack
+		refID := h.raceMu.nextRefID
+		h.raceMu.openRefs[refID] = stack
+		h.raceMu.nextRefID++
 		h.raceMu.Unlock()
+		// In race builds, this closeHook closure will force an allocation.
+		// Race builds are already unperformant (and allocate a stack trace), so
+		// we don't care.
+		closeHook = func() {
+			v.closeHook()
+			h.raceMu.Lock()
+			defer h.raceMu.Unlock()
+			delete(h.raceMu.openRefs, refID)
+		}
 	}
-	return iter, nil
+	return closeHook
 }
 
 // newRangeDelIter is an internal helper that constructs an iterator over a
@@ -736,7 +752,7 @@ func (h *fileCacheHandle) getTableProperties(file *tableMetadata) (*sstable.Prop
 }
 
 type fileCacheValue struct {
-	closeHook func(i any)
+	closeHook func()
 	reader    io.Closer // *sstable.Reader
 	isShared  bool
 
diff --git a/ingest.go b/ingest.go
index e86b76e2..de6b3847 100644
--- a/ingest.go
+++ b/ingest.go
@@ -1270,17 +1270,15 @@ func (d *DB) IngestAndExcise(
 	if d.opts.ReadOnly {
 		return IngestOperationStats{}, ErrReadOnly
 	}
-	if invariants.Enabled {
-		// Excise is only supported on prefix keys.
-		if d.opts.Comparer.Split(exciseSpan.Start) != len(exciseSpan.Start) {
-			panic("IngestAndExcise called with suffixed start key")
-		}
-		if d.opts.Comparer.Split(exciseSpan.End) != len(exciseSpan.End) {
-			panic("IngestAndExcise called with suffixed end key")
-		}
+	// Excise is only supported on prefix keys.
+	if d.opts.Comparer.Split(exciseSpan.Start) != len(exciseSpan.Start) {
+		return IngestOperationStats{}, errors.New("IngestAndExcise called with suffixed start key")
+	}
+	if d.opts.Comparer.Split(exciseSpan.End) != len(exciseSpan.End) {
+		return IngestOperationStats{}, errors.New("IngestAndExcise called with suffixed end key")
 	}
 	if v := d.FormatMajorVersion(); v < FormatMinForSharedObjects {
-		return IngestOperationStats{}, errors.Errorf(
+		return IngestOperationStats{}, errors.Newf(
 			"store has format major version %d; IngestAndExcise requires at least %d",
 			v, FormatMinForSharedObjects,
 		)
@@ -1466,7 +1464,7 @@ func (d *DB) ingest(
 		return IngestOperationStats{}, err
 	}
 
-	if loadResult.fileCount() == 0 {
+	if loadResult.fileCount() == 0 && !exciseSpan.Valid() {
 		// All of the sstables to be ingested were empty. Nothing to do.
 		return IngestOperationStats{}, nil
 	}
@@ -1713,61 +1711,66 @@ func (d *DB) ingest(
 		}
 	}
 
-	info := TableIngestInfo{
-		JobID:     int(jobID),
-		Err:       err,
-		flushable: asFlushable,
-	}
-	if len(loadResult.local) > 0 {
-		info.GlobalSeqNum = loadResult.local[0].SmallestSeqNum
-	} else if len(loadResult.shared) > 0 {
-		info.GlobalSeqNum = loadResult.shared[0].SmallestSeqNum
-	} else {
-		info.GlobalSeqNum = loadResult.external[0].SmallestSeqNum
-	}
+	// TODO(jackson): Refactor this so that the case where there are no files
+	// but a valid excise span is not so exceptional.
+
 	var stats IngestOperationStats
-	if ve != nil {
-		info.Tables = make([]struct {
-			TableInfo
-			Level int
-		}, len(ve.NewTables))
-		for i := range ve.NewTables {
-			e := &ve.NewTables[i]
-			info.Tables[i].Level = e.Level
-			info.Tables[i].TableInfo = e.Meta.TableInfo()
-			stats.Bytes += e.Meta.Size
-			if e.Level == 0 {
-				stats.ApproxIngestedIntoL0Bytes += e.Meta.Size
-			}
-			if metaFlushableOverlaps[e.Meta.FileNum] {
-				stats.MemtableOverlappingFiles++
+	if loadResult.fileCount() > 0 {
+		info := TableIngestInfo{
+			JobID:     int(jobID),
+			Err:       err,
+			flushable: asFlushable,
+		}
+		if len(loadResult.local) > 0 {
+			info.GlobalSeqNum = loadResult.local[0].SmallestSeqNum
+		} else if len(loadResult.shared) > 0 {
+			info.GlobalSeqNum = loadResult.shared[0].SmallestSeqNum
+		} else {
+			info.GlobalSeqNum = loadResult.external[0].SmallestSeqNum
+		}
+		if ve != nil {
+			info.Tables = make([]struct {
+				TableInfo
+				Level int
+			}, len(ve.NewTables))
+			for i := range ve.NewTables {
+				e := &ve.NewTables[i]
+				info.Tables[i].Level = e.Level
+				info.Tables[i].TableInfo = e.Meta.TableInfo()
+				stats.Bytes += e.Meta.Size
+				if e.Level == 0 {
+					stats.ApproxIngestedIntoL0Bytes += e.Meta.Size
+				}
+				if metaFlushableOverlaps[e.Meta.FileNum] {
+					stats.MemtableOverlappingFiles++
+				}
 			}
-		}
-	} else if asFlushable {
-		// NB: If asFlushable == true, there are no shared sstables.
-		info.Tables = make([]struct {
-			TableInfo
-			Level int
-		}, len(loadResult.local))
-		for i, f := range loadResult.local {
-			info.Tables[i].Level = -1
-			info.Tables[i].TableInfo = f.TableInfo()
-			stats.Bytes += f.Size
-			// We don't have exact stats on which files will be ingested into
-			// L0, because actual ingestion into the LSM has been deferred until
-			// flush time. Instead, we infer based on memtable overlap.
-			//
-			// TODO(jackson): If we optimistically compute data overlap (#2112)
-			// before entering the commit pipeline, we can use that overlap to
-			// improve our approximation by incorporating overlap with L0, not
-			// just memtables.
-			if metaFlushableOverlaps[f.FileNum] {
-				stats.ApproxIngestedIntoL0Bytes += f.Size
-				stats.MemtableOverlappingFiles++
+		} else if asFlushable {
+			// NB: If asFlushable == true, there are no shared sstables.
+			info.Tables = make([]struct {
+				TableInfo
+				Level int
+			}, len(loadResult.local))
+			for i, f := range loadResult.local {
+				info.Tables[i].Level = -1
+				info.Tables[i].TableInfo = f.TableInfo()
+				stats.Bytes += f.Size
+				// We don't have exact stats on which files will be ingested into
+				// L0, because actual ingestion into the LSM has been deferred until
+				// flush time. Instead, we infer based on memtable overlap.
+				//
+				// TODO(jackson): If we optimistically compute data overlap (#2112)
+				// before entering the commit pipeline, we can use that overlap to
+				// improve our approximation by incorporating overlap with L0, not
+				// just memtables.
+				if metaFlushableOverlaps[f.FileNum] {
+					stats.ApproxIngestedIntoL0Bytes += f.Size
+					stats.MemtableOverlappingFiles++
+				}
 			}
 		}
+		d.opts.EventListener.TableIngested(info)
 	}
-	d.opts.EventListener.TableIngested(info)
 
 	return stats, err
 }
diff --git a/sstable/reader_iter.go b/sstable/reader_iter.go
index e479c38f..e0a8e8bb 100644
--- a/sstable/reader_iter.go
+++ b/sstable/reader_iter.go
@@ -50,11 +50,9 @@ type Iterator interface {
 	NextPrefix(succKey []byte) *base.InternalKV
 
 	// SetCloseHook sets a function that will be called when the iterator is
-	// closed.  This is used by the file cache to release the reference count on
-	// the open sstable.Reader when the iterator is closed. The closures takes
-	// the iterator as a parameter to enable invariant-build tracking of leaked
-	// iterators.
-	SetCloseHook(fn func(any))
+	// closed. This is used by the file cache to release the reference count on
+	// the open sstable.Reader when the iterator is closed.
+	SetCloseHook(func())
 }
 
 // Iterator positioning optimizations and singleLevelIterator and
diff --git a/sstable/reader_iter_single_lvl.go b/sstable/reader_iter_single_lvl.go
index e50556be..978a6ed2 100644
--- a/sstable/reader_iter_single_lvl.go
+++ b/sstable/reader_iter_single_lvl.go
@@ -71,7 +71,7 @@ type singleLevelIterator[I any, PI indexBlockIterator[I], D any, PD dataBlockIte
 	vbRH         objstorage.ReadHandle
 	vbRHPrealloc objstorageprovider.PreallocatedReadHandle
 	err          error
-	closeHook    func(i any)
+	closeHook    func()
 
 	readBlockEnv block.ReadEnv
 
@@ -1515,9 +1515,8 @@ func (i *singleLevelIterator[I, PI, D, PD]) Error() error {
 
 // SetCloseHook sets a function that will be called when the iterator is closed.
 // This is used by the file cache to release the reference count on the open
-// sstable.Reader when the iterator is closed. The closures takes the iterator
-// as a parameter to enable invariant-build tracking of leaked iterators.
-func (i *singleLevelIterator[I, PI, D, PD]) SetCloseHook(fn func(i any)) {
+// sstable.Reader when the iterator is closed.
+func (i *singleLevelIterator[I, PI, D, PD]) SetCloseHook(fn func()) {
 	i.closeHook = fn
 }
 
@@ -1546,7 +1545,7 @@ func (i *singleLevelIterator[I, PI, D, PD]) closeInternal() error {
 	}
 
 	if i.closeHook != nil {
-		i.closeHook(i)
+		i.closeHook()
 	}
 	var err error
 	err = firstError(err, PD(&i.data).Close())
diff --git a/sstable/reader_iter_two_lvl.go b/sstable/reader_iter_two_lvl.go
index 48e6db7d..590acf84 100644
--- a/sstable/reader_iter_two_lvl.go
+++ b/sstable/reader_iter_two_lvl.go
@@ -1015,7 +1015,7 @@ func (i *twoLevelIterator[I, PI, D, PD]) SetContext(ctx context.Context) {
 	i.secondLevel.SetContext(ctx)
 }
 
-func (i *twoLevelIterator[I, PI, D, PD]) SetCloseHook(fn func(i any)) {
+func (i *twoLevelIterator[I, PI, D, PD]) SetCloseHook(fn func()) {
 	i.secondLevel.SetCloseHook(fn)
 }
 


Test patch:
diff --git a/checkpoint_test.go b/checkpoint_test.go
index 112d37b7..7f1a16c7 100644
--- a/checkpoint_test.go
+++ b/checkpoint_test.go
@@ -108,7 +108,7 @@ func testCheckpointImpl(t *testing.T, ddFile string, createOnShared bool) {
 
 			// Hacky but the command doesn't expect a db string. Get rid of it.
 			td.CmdArgs = td.CmdArgs[1:]
-			if err := runIngestAndExciseCmd(td, d, mem); err != nil {
+			if err := runIngestAndExciseCmd(td, d); err != nil {
 				return err.Error()
 			}
 			return ""
diff --git a/compaction_picker_test.go b/compaction_picker_test.go
index 9667aa4b..9571ae94 100644
--- a/compaction_picker_test.go
+++ b/compaction_picker_test.go
@@ -1386,7 +1386,7 @@ func TestCompactionPickerPickFile(t *testing.T) {
 			return ""
 
 		case "ingest-and-excise":
-			if err := runIngestAndExciseCmd(td, d, d.opts.FS); err != nil {
+			if err := runIngestAndExciseCmd(td, d); err != nil {
 				return err.Error()
 			}
 			return ""
diff --git a/data_test.go b/data_test.go
index 87b6ec70..58e2e53c 100644
--- a/data_test.go
+++ b/data_test.go
@@ -1252,7 +1252,18 @@ func (d *DB) waitTableStats() {
 	}
 }
 
-func runIngestAndExciseCmd(td *datadriven.TestData, d *DB, fs vfs.FS) error {
+func runExciseCmd(td *datadriven.TestData, d *DB) error {
+	if len(td.CmdArgs) != 2 {
+		return errors.New("excise expects two arguments: <start-key> <end-key>")
+	}
+	exciseSpan := KeyRange{
+		Start: []byte(td.CmdArgs[0].String()),
+		End:   []byte(td.CmdArgs[1].String()),
+	}
+	return d.Excise(context.Background(), exciseSpan)
+}
+
+func runIngestAndExciseCmd(td *datadriven.TestData, d *DB) error {
 	var exciseSpan KeyRange
 	paths := make([]string, 0, len(td.CmdArgs))
 	for i, arg := range td.CmdArgs {
diff --git a/db_test.go b/db_test.go
index f3f3bbce..6a13f1df 100644
--- a/db_test.go
+++ b/db_test.go
@@ -2198,6 +2198,13 @@ func TestDeterminism(t *testing.T) {
 					require.NoError(t, runBuildCmd(td, d, fs))
 					return ""
 				})
+			case "excise":
+				return addStep(td, func(td *datadriven.TestData) string {
+					if err := runExciseCmd(td, d); err != nil {
+						return err.Error()
+					}
+					return ""
+				})
 			case "flush":
 				return addStep(td, func(td *datadriven.TestData) string {
 					_, err := d.AsyncFlush()
@@ -2208,7 +2215,7 @@ func TestDeterminism(t *testing.T) {
 				})
 			case "ingest-and-excise":
 				return addStep(td, func(td *datadriven.TestData) string {
-					if err := runIngestAndExciseCmd(td, d, fs); err != nil {
+					if err := runIngestAndExciseCmd(td, d); err != nil {
 						return err.Error()
 					}
 					return ""
diff --git a/ingest_test.go b/ingest_test.go
index c12e1797..a977a8c8 100644
--- a/ingest_test.go
+++ b/ingest_test.go
@@ -721,25 +721,21 @@ func TestExcise(t *testing.T) {
 				return err.Error()
 			}
 			return ""
-
 		case "flush":
 			if err := d.Flush(); err != nil {
 				return err.Error()
 			}
 			return ""
-
 		case "block-flush":
 			d.mu.Lock()
 			d.mu.compact.flushing = true
 			d.mu.Unlock()
 			return ""
-
 		case "allow-flush":
 			d.mu.Lock()
 			d.mu.compact.flushing = false
 			d.mu.Unlock()
 			return ""
-
 		case "ingest":
 			noWait := td.HasArg("no-wait")
 			if !noWait {
@@ -761,7 +757,6 @@ func TestExcise(t *testing.T) {
 				return "memtable flushed"
 			}
 			return ""
-
 		case "ingest-and-excise":
 			var prevFlushableIngests uint64
 			noWait := td.HasArg("no-wait")
@@ -772,7 +767,7 @@ func TestExcise(t *testing.T) {
 				d.mu.Unlock()
 			}
 
-			if err := runIngestAndExciseCmd(td, d, mem); err != nil {
+			if err := runIngestAndExciseCmd(td, d); err != nil {
 				return err.Error()
 			}
 			if noWait {
@@ -792,7 +787,6 @@ func TestExcise(t *testing.T) {
 				return "memtable flushed"
 			}
 			return ""
-
 		case "file-only-snapshot":
 			if len(td.CmdArgs) != 1 {
 				panic("insufficient args for file-only-snapshot command")
@@ -811,10 +805,8 @@ func TestExcise(t *testing.T) {
 			s := d.NewEventuallyFileOnlySnapshot(keyRanges)
 			efos[name] = s
 			return "ok"
-
 		case "get":
 			return runGetCmd(t, td, d)
-
 		case "iter":
 			opts := &IterOptions{
 				KeyTypes: IterKeyTypePointsAndRanges,
@@ -849,10 +841,8 @@ func TestExcise(t *testing.T) {
 			}
 			iter, _ := reader.NewIter(opts)
 			return runIterCmd(td, iter, true)
-
 		case "lsm":
 			return runLSMCmd(td, d)
-
 		case "metrics":
 			// The asynchronous loading of table stats can change metrics, so
 			// wait for all the tables' stats to be loaded.
@@ -861,11 +851,14 @@ func TestExcise(t *testing.T) {
 			d.mu.Unlock()
 
 			return d.Metrics().StringForTests()
-
 		case "wait-pending-table-stats":
 			return runTableStatsCmd(td, d)
-
 		case "excise":
+			if err := runExciseCmd(td, d); err != nil {
+				return err.Error()
+			}
+			return ""
+		case "excise-dryrun":
 			ve := &versionEdit{
 				DeletedTables: map[deletedFileEntry]*tableMetadata{},
 			}
@@ -894,7 +887,6 @@ func TestExcise(t *testing.T) {
 			d.mu.versions.logUnlock()
 			d.mu.Unlock()
 			return fmt.Sprintf("would excise %d files, use ingest-and-excise to excise.\n%s", len(ve.DeletedTables), ve.DebugString(base.DefaultFormatter))
-
 		case "confirm-backing":
 			// Confirms that the files have the same FileBacking.
 			fileNums := make(map[base.FileNum]struct{})
@@ -1097,7 +1089,7 @@ func testIngestSharedImpl(
 			return ""
 
 		case "ingest-and-excise":
-			if err := runIngestAndExciseCmd(td, d, d.opts.FS); err != nil {
+			if err := runIngestAndExciseCmd(td, d); err != nil {
 				return err.Error()
 			}
 			// Wait for a possible flush.
@@ -1595,7 +1587,7 @@ func TestConcurrentExcise(t *testing.T) {
 			d.mu.Lock()
 			prevFlushableIngests := d.mu.versions.metrics.Flush.AsIngestCount
 			d.mu.Unlock()
-			if err := runIngestAndExciseCmd(td, d, d.opts.FS); err != nil {
+			if err := runIngestAndExciseCmd(td, d); err != nil {
 				return err.Error()
 			}
 			// Wait for a possible flush.
diff --git a/metrics_test.go b/metrics_test.go
index de7cc4ee..c3b77610 100644
--- a/metrics_test.go
+++ b/metrics_test.go
@@ -279,7 +279,7 @@ func TestMetrics(t *testing.T) {
 			return s
 
 		case "ingest-and-excise":
-			if err := runIngestAndExciseCmd(td, d, d.opts.FS); err != nil {
+			if err := runIngestAndExciseCmd(td, d); err != nil {
 				return err.Error()
 			}
 			return ""
diff --git a/table_stats_test.go b/table_stats_test.go
index 8409a884..7a320b6e 100644
--- a/table_stats_test.go
+++ b/table_stats_test.go
@@ -142,7 +142,7 @@ func TestTableStats(t *testing.T) {
 			return ""
 
 		case "ingest-and-excise":
-			if err := runIngestAndExciseCmd(td, d, d.opts.FS); err != nil {
+			if err := runIngestAndExciseCmd(td, d); err != nil {
 				return err.Error()
 			}
 			// Wait for a possible flush.
diff --git a/testdata/determinism b/testdata/determinism
index e1026ba0..e998d00f 100644
--- a/testdata/determinism
+++ b/testdata/determinism
@@ -134,3 +134,12 @@ run count=100
 sequential( 0:define 1:memtable-info 2:batch 3:batch 4:memtable-info parallel( 5:batch 6:batch ) )
 ----
 ok
+
+excise a z
+----
+7:excise
+
+run count=100
+sequential( 0:define 1:memtable-info 2:batch 3:batch 4:memtable-info parallel( 5:batch 6:batch ) 7:excise )
+----
+ok
diff --git a/testdata/excise b/testdata/excise
index fa614a91..7a42c881 100644
--- a/testdata/excise
+++ b/testdata/excise
@@ -28,7 +28,7 @@ L0.0:
 L6:
   000004:[a#10,SET-l#10,SET]
 
-excise c k
+excise-dryrun c k
 ----
 would excise 2 files, use ingest-and-excise to excise.
   del-table:     L0 000006
@@ -38,7 +38,7 @@ would excise 2 files, use ingest-and-excise to excise.
   add-backing:   000004
 
 
-excise a e
+excise-dryrun a e
 ----
 would excise 2 files, use ingest-and-excise to excise.
   del-table:     L0 000006
@@ -48,7 +48,7 @@ would excise 2 files, use ingest-and-excise to excise.
   add-backing:   000006
   add-backing:   000004
 
-excise e z
+excise-dryrun e z
 ----
 would excise 2 files, use ingest-and-excise to excise.
   del-table:     L0 000006
@@ -58,7 +58,7 @@ would excise 2 files, use ingest-and-excise to excise.
   add-backing:   000006
   add-backing:   000004
 
-excise f l
+excise-dryrun f l
 ----
 would excise 2 files, use ingest-and-excise to excise.
   del-table:     L0 000006
@@ -69,7 +69,7 @@ would excise 2 files, use ingest-and-excise to excise.
   add-backing:   000006
   add-backing:   000004
 
-excise f ll
+excise-dryrun f ll
 ----
 would excise 2 files, use ingest-and-excise to excise.
   del-table:     L0 000006
@@ -79,7 +79,7 @@ would excise 2 files, use ingest-and-excise to excise.
   add-backing:   000006
   add-backing:   000004
 
-excise p q
+excise-dryrun p q
 ----
 would excise 0 files, use ingest-and-excise to excise.
 
@@ -159,7 +159,7 @@ L0.0:
 L6:
   000004:[c#10,RANGEKEYSET-f#inf,RANGEKEYSET]
 
-excise f g
+excise-dryrun f g
 ----
 would excise 1 files, use ingest-and-excise to excise.
   del-table:     L0 000005
@@ -167,21 +167,21 @@ would excise 1 files, use ingest-and-excise to excise.
   add-table:     L0 000007(000005):[g#11,RANGEDEL-i#inf,RANGEDEL] seqnums:[11-11] points:[g#11,RANGEDEL-i#inf,RANGEDEL] size:1
   add-backing:   000005
 
-excise b c
+excise-dryrun b c
 ----
 would excise 1 files, use ingest-and-excise to excise.
   del-table:     L0 000005
   add-table:     L0 000008(000005):[g#11,RANGEDEL-i#inf,RANGEDEL] seqnums:[11-11] points:[g#11,RANGEDEL-i#inf,RANGEDEL] size:1
   add-backing:   000005
 
-excise i j
+excise-dryrun i j
 ----
 would excise 0 files, use ingest-and-excise to excise.
 
 # Excise mid range key. This will not happen in practice, but excise()
 # supports it.
 
-excise c d
+excise-dryrun c d
 ----
 would excise 2 files, use ingest-and-excise to excise.
   del-table:     L0 000005
@@ -709,6 +709,37 @@ c: (somethingElse, .)
 .
 .
 
+excise aa ab
+----
+
+lsm
+----
+L0.0:
+  000019:[a#22,SET-a#22,SET]
+  000020:[b#23,SET-f#23,SET]
+L6:
+  000023(000018):[a#0,SET-a#0,SET]
+  000022(000018):[f#0,SET-x#0,SET]
+
+excise g h
+----
+
+lsm
+----
+L0.0:
+  000019:[a#22,SET-a#22,SET]
+  000020:[b#23,SET-f#23,SET]
+L6:
+  000023(000018):[a#0,SET-a#0,SET]
+  000024(000018):[f#0,SET-f#0,SET]
+  000025(000018):[x#0,SET-x#0,SET]
+
+excise a z
+----
+
+lsm
+----
+
 # Two overlapping ingestions wait on one another even if
 # the overlap is only on the excise span.