Repository: cockroachdb/pebble
Pull request #3607 (2024-05-10 15:30:55)

objstorageprovider: small readahead improvements
- The limit was not being updated in certain cases, which delayed when readahead would occur. This is fixed, and new test case exercises this.
- The code in recordCacheHit is almost identical to the one in maybeReadahead and the main comprehension challenge of the code is the various predicates (and their code comments with examples), which are repeated (without the examples). These are now merged into a single helper method.
- A small side-effect of the previous change is that numReads is incremented on a cache hit when we are already above the threshold. This has no actual effect on when readahead will happen. And arguably, this new behavior is more principled.
- One of the test cases had a missing - and was not being exercised, and resulting in an incorrect error in a later test case. This is fixed.
- There is additional invariant documentation.

cockroach-teamcity: 

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/cockroachdb/pebble/3607)
<!-- Reviewable:end -->


Patch:
diff --git a/compaction.go b/compaction.go
index bccd3bdf..61c19f68 100644
--- a/compaction.go
+++ b/compaction.go
@@ -18,7 +18,6 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/compact"
-	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/keyspan/keyspanimpl"
 	"github.com/cockroachdb/pebble/internal/manifest"
@@ -2163,14 +2162,26 @@ func (d *DB) compact1(c *compaction, errChannel chan error) (err error) {
 // d.mu must be held when calling this method. The mutex will be released when
 // doing IO.
 func (d *DB) runCopyCompaction(
-	jobID JobID,
-	c *compaction,
-	inputMeta *fileMetadata,
-	objMeta objstorage.ObjectMetadata,
-	ve *versionEdit,
-) error {
-	ctx := context.TODO()
+	jobID JobID, c *compaction,
+) (ve *versionEdit, stats compact.Stats, _ error) {
+	iter := c.startLevel.files.Iter()
+	inputMeta := iter.First()
+	if iter.Next() != nil {
+		return nil, compact.Stats{}, base.AssertionFailedf("got more than one file for a move compaction")
+	}
+	if c.cancel.Load() {
+		return nil, compact.Stats{}, ErrCancelledCompaction
+	}
+	ve = &versionEdit{
+		DeletedFiles: map[deletedFileEntry]*fileMetadata{
+			{Level: c.startLevel.level, FileNum: inputMeta.FileNum}: inputMeta,
+		},
+	}
 
+	objMeta, err := d.objProvider.Lookup(fileTypeTable, inputMeta.FileBacking.DiskFileNum)
+	if err != nil {
+		return nil, compact.Stats{}, err
+	}
 	if !objMeta.IsExternal() {
 		if objMeta.IsRemote() || !remote.ShouldCreateShared(d.opts.Experimental.CreateOnShared, c.outputLevel.level) {
 			panic("pebble: scheduled a copy compaction that is not actually moving files to shared storage")
@@ -2215,14 +2226,6 @@ func (d *DB) runCopyCompaction(
 		newMeta.InitPhysicalBacking()
 	}
 
-	c.metrics = map[int]*LevelMetrics{
-		c.outputLevel.level: {
-			BytesIn:         inputMeta.Size,
-			BytesCompacted:  inputMeta.Size,
-			TablesCompacted: 1,
-		},
-	}
-
 	// Before dropping the db mutex, grab a ref to the current version. This
 	// prevents any concurrent excises from deleting files that this compaction
 	// needs to read/maintain a reference to.
@@ -2244,11 +2247,12 @@ func (d *DB) runCopyCompaction(
 
 	// If the src obj is external, we're doing an external to local/shared copy.
 	if objMeta.IsExternal() {
+		ctx := context.TODO()
 		src, err := d.objProvider.OpenForReading(
 			ctx, fileTypeTable, inputMeta.FileBacking.DiskFileNum, objstorage.OpenOptions{},
 		)
 		if err != nil {
-			return err
+			return nil, compact.Stats{}, err
 		}
 		defer func() {
 			if src != nil {
@@ -2263,7 +2267,7 @@ func (d *DB) runCopyCompaction(
 			},
 		)
 		if err != nil {
-			return err
+			return nil, compact.Stats{}, err
 		}
 		deleteOnExit = true
 
@@ -2287,7 +2291,17 @@ func (d *DB) runCopyCompaction(
 		)
 		src = nil // We passed src to CopySpan; it's responsible for closing it.
 		if err != nil {
-			return err
+			if errors.Is(err, sstable.ErrEmptySpan) {
+				// The virtual table was empty. Just remove the backing file.
+				// Note that deleteOnExit is true so we will delete the created object.
+				c.metrics = map[int]*LevelMetrics{
+					c.outputLevel.level: {
+						BytesIn: inputMeta.Size,
+					},
+				}
+				return ve, compact.Stats{}, nil
+			}
+			return nil, compact.Stats{}, err
 		}
 		newMeta.FileBacking.Size = wrote
 		newMeta.Size = wrote
@@ -2295,22 +2309,31 @@ func (d *DB) runCopyCompaction(
 		_, err := d.objProvider.LinkOrCopyFromLocal(context.TODO(), d.opts.FS,
 			d.objProvider.Path(objMeta), fileTypeTable, newMeta.FileBacking.DiskFileNum,
 			objstorage.CreateOptions{PreferSharedStorage: true})
-
 		if err != nil {
-			return err
+			return nil, compact.Stats{}, err
 		}
 		deleteOnExit = true
 	}
-	ve.NewFiles[0].Meta = newMeta
+	ve.NewFiles = []newFileEntry{{
+		Level: c.outputLevel.level,
+		Meta:  newMeta,
+	}}
 	if newMeta.Virtual {
 		ve.CreatedBackingTables = []*fileBacking{newMeta.FileBacking}
 	}
+	c.metrics = map[int]*LevelMetrics{
+		c.outputLevel.level: {
+			BytesIn:         inputMeta.Size,
+			BytesCompacted:  newMeta.Size,
+			TablesCompacted: 1,
+		},
+	}
 
 	if err := d.objProvider.Sync(); err != nil {
-		return err
+		return nil, compact.Stats{}, err
 	}
 	deleteOnExit = false
-	return nil
+	return ve, compact.Stats{}, nil
 }
 
 func (d *DB) runDeleteOnlyCompaction(
@@ -2334,23 +2357,17 @@ func (d *DB) runDeleteOnlyCompaction(
 	return ve, stats, nil
 }
 
-func (d *DB) runMoveOrCopyCompaction(
+func (d *DB) runMoveCompaction(
 	jobID JobID, c *compaction,
 ) (ve *versionEdit, stats compact.Stats, _ error) {
 	iter := c.startLevel.files.Iter()
 	meta := iter.First()
-	if invariants.Enabled {
-		if iter.Next() != nil {
-			panic("got more than one file for a move or copy compaction")
-		}
+	if iter.Next() != nil {
+		return nil, stats, base.AssertionFailedf("got more than one file for a move compaction")
 	}
 	if c.cancel.Load() {
 		return ve, stats, ErrCancelledCompaction
 	}
-	objMeta, err := d.objProvider.Lookup(fileTypeTable, meta.FileBacking.DiskFileNum)
-	if err != nil {
-		return ve, stats, err
-	}
 	c.metrics = map[int]*LevelMetrics{
 		c.outputLevel.level: {
 			BytesMoved:  meta.Size,
@@ -2365,12 +2382,8 @@ func (d *DB) runMoveOrCopyCompaction(
 			{Level: c.outputLevel.level, Meta: meta},
 		},
 	}
-	if c.kind == compactionKindMove {
-		return ve, stats, nil
-	}
 
-	err = d.runCopyCompaction(jobID, c, meta, objMeta, ve)
-	return ve, stats, err
+	return ve, stats, nil
 }
 
 // runCompaction runs a compaction that produces new on-disk tables from
@@ -2386,8 +2399,10 @@ func (d *DB) runCompaction(
 	switch c.kind {
 	case compactionKindDeleteOnly:
 		return d.runDeleteOnlyCompaction(jobID, c)
-	case compactionKindMove, compactionKindCopy:
-		return d.runMoveOrCopyCompaction(jobID, c)
+	case compactionKindMove:
+		return d.runMoveCompaction(jobID, c)
+	case compactionKindCopy:
+		return d.runCopyCompaction(jobID, c)
 	case compactionKindIngestedFlushable:
 		panic("pebble: runCompaction cannot handle compactionKindIngestedFlushable.")
 	}
diff --git a/ingest.go b/ingest.go
index 63cc4523..c12591b2 100644
--- a/ingest.go
+++ b/ingest.go
@@ -698,68 +698,6 @@ func (d *DB) ingestAttachRemote(jobID JobID, lr ingestLoadResult) error {
 		}
 	}
 
-	if invariants.Enabled || d.opts.Experimental.CheckExternalIngestions {
-		// Verify that the ingestion is not empty. Empty tables cause correctness
-		// issues in some iterators.
-		for i := range lr.external {
-			e := &lr.external[i]
-			isEmpty, err := func() (bool, error) {
-				readable, err := d.objProvider.OpenForReading(
-					context.Background(), fileTypeTable, e.FileBacking.DiskFileNum, objstorage.OpenOptions{},
-				)
-				if err != nil {
-					return false, err
-				}
-				reader, err := sstable.NewReader(readable, d.opts.MakeReaderOptions())
-				if err != nil {
-					readable.Close()
-					return false, err
-				}
-				defer reader.Close()
-				iter, err := reader.NewIter(e.IterTransforms(), nil, nil)
-				if err != nil {
-					return false, err
-				}
-				if iter != nil {
-					defer iter.Close()
-					if kv := iter.SeekGE(e.external.StartKey, base.SeekGEFlagsNone); kv != nil {
-						cmp := d.opts.Comparer.Compare(kv.K.UserKey, e.external.EndKey)
-						if cmp < 0 || (cmp == 0 && e.external.EndKeyIsInclusive) {
-							return false, nil
-						}
-					}
-					if err := iter.Error(); err != nil {
-						return false, err
-					}
-				}
-				rangeIter, err := reader.NewRawRangeDelIter(e.IterTransforms())
-				if err != nil {
-					return false, err
-				}
-				if rangeIter != nil {
-					defer rangeIter.Close()
-					span, err := rangeIter.SeekGE(e.external.StartKey)
-					if err != nil {
-						return false, err
-					}
-					if span != nil {
-						cmp := d.opts.Comparer.Compare(span.Start, e.external.EndKey)
-						if cmp < 0 || (cmp == 0 && e.external.EndKeyIsInclusive) {
-							return false, nil
-						}
-					}
-				}
-				return true, nil
-			}()
-			if err != nil {
-				return err
-			}
-			if isEmpty {
-				panic(fmt.Sprintf("external ingestion is empty: %s (%q %q)", e.external.ObjName, e.external.StartKey, e.external.EndKey))
-			}
-		}
-	}
-
 	if d.opts.EventListener.TableCreated != nil {
 		for i := range remoteObjMetas {
 			d.opts.EventListener.TableCreated(TableCreateInfo{
diff --git a/internal/base/iterator.go b/internal/base/iterator.go
index 7cbea1b7..7531e5fb 100644
--- a/internal/base/iterator.go
+++ b/internal/base/iterator.go
@@ -107,13 +107,15 @@ type InternalIterator interface {
 	// SeekPrefixGE only checks the upper bound. It is up to the caller to ensure
 	// that key is greater than or equal to the lower bound.
 	//
-	// The prefix argument is used by some InternalIterator implementations (e.g.
-	// sstable.Reader) to avoid expensive operations. This operation is only
-	// useful when a user-defined Split function is supplied to the Comparer for
-	// the DB. The supplied prefix will be the prefix of the given key returned by
-	// that Split function. If the iterator is able to determine that no key with
-	// the prefix exists, it can return (nil,nilv). Unlike SeekGE, this is not an
-	// indication that iteration is exhausted.
+	// The prefix argument is used by some InternalIterator implementations
+	// (e.g.  sstable.Reader) to avoid expensive operations. This operation is
+	// only useful when a user-defined Split function is supplied to the
+	// Comparer for the DB. The supplied prefix will be the prefix of the given
+	// key returned by that Split function. If the iterator is able to determine
+	// that no key with the prefix exists, it can return (nil,nilv). Unlike
+	// SeekGE, this is not an indication that iteration is exhausted. The prefix
+	// byte slice is guaranteed to be stable until the next absolute positioning
+	// operation.
 	//
 	// Note that the iterator may return keys not matching the prefix. It is up
 	// to the caller to check if the prefix matches.
diff --git a/level_iter.go b/level_iter.go
index b799dd60..00af0887 100644
--- a/level_iter.go
+++ b/level_iter.go
@@ -31,21 +31,13 @@ type internalIterOpts struct {
 //
 // levelIter is used during compaction and as part of the Iterator
 // implementation. When used as part of the Iterator implementation, level
-// iteration needs to "pause" at sstable boundaries if a range deletion iterator
-// is open. In this case, we materialize a "synthetic" InternalKV to return from
-// levelIter. This prevents mergingIter from advancing past the sstable until
-// the sstable contains the smallest (or largest for reverse iteration) key in
-// the merged heap. Note that mergingIter treats a range deletion tombstone
-// returned by the point iterator as a no-op.
-//
-// SeekPrefixGE presents the need for a second type of pausing. If an sstable
-// iterator returns "not found" for a SeekPrefixGE operation, we don't want to
-// advance to the next sstable as the "not found" does not indicate that all of
-// the keys in the sstable are less than the search key. Advancing to the next
-// sstable would cause us to skip over range tombstones, violating
-// correctness. Instead, SeekPrefixGE creates a synthetic boundary key with the
-// kind InternalKeyKindRangeDeletion which will be used to pause the levelIter
-// at the sstable until the mergingIter is ready to advance past it.
+// iteration needs to "pause" at range deletion boundaries if file contains
+// range deletions. In this case, the levelIter uses a keyspan.InterleavingIter
+// to materialize InternalKVs at start and end boundaries of range deletions.
+// This prevents mergingIter from advancing past the sstable until the sstable
+// contains the smallest (or largest for reverse iteration) key in the merged
+// heap. Note that mergingIter treats a range deletion tombstone returned by the
+// point iterator as a no-op.
 type levelIter struct {
 	// The context is stored here since (a) iterators are expected to be
 	// short-lived (since they pin sstables), (b) plumbing a context into every
@@ -60,6 +52,9 @@ type levelIter struct {
 	// recent call to SetBounds.
 	lower []byte
 	upper []byte
+	// prefix holds the iteration prefix when the most recent absolute
+	// positioning method was a SeekPrefixGE.
+	prefix []byte
 	// The iterator options for the currently open table. If
 	// tableOpts.{Lower,Upper}Bound are nil, the corresponding iteration boundary
 	// does not lie within the table bounds.
@@ -161,6 +156,7 @@ func (l *levelIter) init(
 	l.err = nil
 	l.level = level
 	l.logger = opts.getLogger()
+	l.prefix = nil
 	l.lower = opts.LowerBound
 	l.upper = opts.UpperBound
 	l.tableOpts.TableFilter = opts.TableFilter
@@ -629,6 +625,7 @@ func (l *levelIter) SeekGE(key []byte, flags base.SeekGEFlags) *base.InternalKV
 
 	l.err = nil // clear cached iteration error
 	l.exhaustedDir = 0
+	l.prefix = nil
 	// NB: the top-level Iterator has already adjusted key based on
 	// IterOptions.LowerBound.
 	loadFileIndicator := l.loadFile(l.findFileGE(key, flags), +1)
@@ -651,9 +648,9 @@ func (l *levelIter) SeekPrefixGE(prefix, key []byte, flags base.SeekGEFlags) *ba
 	if invariants.Enabled && l.lower != nil && l.cmp(key, l.lower) < 0 {
 		panic(errors.AssertionFailedf("levelIter SeekGE to key %q violates lower bound %q", key, l.lower))
 	}
-
 	l.err = nil // clear cached iteration error
 	l.exhaustedDir = 0
+	l.prefix = prefix
 
 	// NB: the top-level Iterator has already adjusted key based on
 	// IterOptions.LowerBound.
@@ -673,19 +670,6 @@ func (l *levelIter) SeekPrefixGE(prefix, key []byte, flags base.SeekGEFlags) *ba
 	if err := l.iter.Error(); err != nil {
 		return nil
 	}
-	// It is possible that we are here because bloom filter matching failed. In
-	// that case it is likely that all keys matching the prefix are wholly
-	// within the current file and cannot be in the subsequent file. In that
-	// case we don't want to go to the next file, since loading and seeking in
-	// there has some cost. Additionally, for sparse key spaces, loading the
-	// next file will defeat the optimization for the next SeekPrefixGE that is
-	// called with flags.TrySeekUsingNext(), since for sparse key spaces it is
-	// likely that the next key will also be contained in the current file.
-	n := l.split(l.iterFile.LargestPointKey.UserKey)
-	if l.cmp(prefix, l.iterFile.LargestPointKey.UserKey[:n]) < 0 {
-		l.exhaustedForward()
-		return nil
-	}
 	return l.verify(l.skipEmptyFileForward())
 }
 
@@ -696,6 +680,7 @@ func (l *levelIter) SeekLT(key []byte, flags base.SeekLTFlags) *base.InternalKV
 
 	l.err = nil // clear cached iteration error
 	l.exhaustedDir = 0
+	l.prefix = nil
 
 	// NB: the top-level Iterator has already adjusted key based on
 	// IterOptions.UpperBound.
@@ -716,6 +701,7 @@ func (l *levelIter) First() *base.InternalKV {
 
 	l.err = nil // clear cached iteration error
 	l.exhaustedDir = 0
+	l.prefix = nil
 
 	// NB: the top-level Iterator will call SeekGE if IterOptions.LowerBound is
 	// set.
@@ -736,6 +722,7 @@ func (l *levelIter) Last() *base.InternalKV {
 
 	l.err = nil // clear cached iteration error
 	l.exhaustedDir = 0
+	l.prefix = nil
 
 	// NB: the top-level Iterator will call SeekLT if IterOptions.UpperBound is
 	// set.
@@ -835,6 +822,27 @@ func (l *levelIter) skipEmptyFileForward() *base.InternalKV {
 			l.exhaustedForward()
 			return nil
 		}
+
+		// If the iterator is in prefix iteration mode, it's possible that we
+		// are here because bloom filter matching failed. In that case it is
+		// likely that all keys matching the prefix are wholly within the
+		// current file and cannot be in a subsequent file. In that case we
+		// don't want to go to the next file, since loading and seeking in there
+		// has some cost.
+		//
+		// This is not just an optimization. We must not advance to the next
+		// file if the current file might possibly contain keys relevant to any
+		// prefix greater than our current iteration prefix. If we did, a
+		// subsequent SeekPrefixGE with TrySeekUsingNext could mistakenly skip
+		// the file's relevant keys.
+		if l.prefix != nil {
+			fileLargestPrefix := l.iterFile.LargestPointKey.UserKey[:l.split(l.iterFile.LargestPointKey.UserKey)]
+			if l.cmp(fileLargestPrefix, l.prefix) > 0 {
+				l.exhaustedForward()
+				return nil
+			}
+		}
+
 		// Current file was exhausted. Move to the next file.
 		if l.loadFile(l.files.Next(), +1) == noFileLoaded {
 			l.exhaustedForward()
diff --git a/metamorphic/build.go b/metamorphic/build.go
index c4e232b6..623c6267 100644
--- a/metamorphic/build.go
+++ b/metamorphic/build.go
@@ -274,32 +274,6 @@ func openExternalObj(
 	return reader, pointIter, rangeDelIter, rangeKeyIter
 }
 
-// externalObjIsEmpty returns true if the given external object has no point or
-// range keys withing the given bounds.
-func externalObjIsEmpty(
-	t *Test, externalObjID objID, bounds pebble.KeyRange, syntheticPrefix sstable.SyntheticPrefix,
-) bool {
-	reader, pointIter, rangeDelIter, rangeKeyIter := openExternalObj(t, externalObjID, bounds, syntheticPrefix)
-	defer reader.Close()
-	defer closeIters(pointIter, rangeDelIter, rangeKeyIter)
-
-	kv := pointIter.First()
-	panicIfErr(pointIter.Error())
-	if kv != nil {
-		return false
-	}
-	for _, it := range []keyspan.FragmentIterator{rangeDelIter, rangeKeyIter} {
-		if it != nil {
-			span, err := it.First()
-			panicIfErr(err)
-			if span != nil {
-				return false
-			}
-		}
-	}
-	return true
-}
-
 func panicIfErr(err error) {
 	if err != nil {
 		panic(err)
diff --git a/metamorphic/ops.go b/metamorphic/ops.go
index 07299334..0db675fc 100644
--- a/metamorphic/ops.go
+++ b/metamorphic/ops.go
@@ -995,33 +995,12 @@ type externalObjWithBounds struct {
 func (o *ingestExternalFilesOp) run(t *Test, h historyRecorder) {
 	db := t.getDB(o.dbID)
 
-	// We modify objs to eliminate empty objects.
-	var objs []externalObjWithBounds
-	for _, obj := range o.objs {
-		// Make sure the object exists and is not empty.
-		objMeta := t.getExternalObj(obj.externalObjID)
-		if m := objMeta.sstMeta; !m.HasPointKeys && !m.HasRangeKeys && !m.HasRangeDelKeys {
-			continue
-		}
-		if externalObjIsEmpty(t, obj.externalObjID, obj.bounds, obj.syntheticPrefix) {
-			// Currently we don't support ingesting external objects that have no point
-			// or range keys in the given range. Filter out any such objects.
-			// TODO(radu): even though we don't expect this case in practice, eventually
-			// we want to make sure that it doesn't cause failures.
-			continue
-		}
-		objs = append(objs, obj)
-	}
-	if len(objs) == 0 {
-		h.Recordf("%s // no-op", o)
-		return
-	}
 	var err error
 	if !t.testOpts.externalStorageEnabled {
 		// Emulate the operation by crating local, truncated SST files and ingesting
 		// them.
 		var paths []string
-		for i, obj := range objs {
+		for i, obj := range o.objs {
 			// Make sure the object exists and is not empty.
 			path, sstMeta := buildForIngestExternalEmulation(
 				t, o.dbID, obj.externalObjID, obj.bounds, obj.syntheticSuffix, obj.syntheticPrefix, i,
@@ -1034,8 +1013,8 @@ func (o *ingestExternalFilesOp) run(t *Test, h historyRecorder) {
 			err = db.Ingest(paths)
 		}
 	} else {
-		external := make([]pebble.ExternalFile, len(objs))
-		for i, obj := range objs {
+		external := make([]pebble.ExternalFile, len(o.objs))
+		for i, obj := range o.objs {
 			meta := t.getExternalObj(obj.externalObjID)
 			external[i] = pebble.ExternalFile{
 				Locator:           "external",
diff --git a/metrics.go b/metrics.go
index 8b9ff258..a19bf2b9 100644
--- a/metrics.go
+++ b/metrics.go
@@ -157,6 +157,7 @@ type Metrics struct {
 		DefaultCount      int64
 		DeleteOnlyCount   int64
 		ElisionOnlyCount  int64
+		CopyCount         int64
 		MoveCount         int64
 		ReadCount         int64
 		RewriteCount      int64
@@ -569,13 +570,14 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
 		redact.Safe(m.Compact.NumInProgress),
 		humanize.Bytes.Int64(m.Compact.InProgressBytes))
 
-	w.Printf("             default: %d  delete: %d  elision: %d  move: %d  read: %d  rewrite: %d  multi-level: %d\n",
+	w.Printf("             default: %d  delete: %d  elision: %d  move: %d  read: %d  rewrite: %d  copy: %d  multi-level: %d\n",
 		redact.Safe(m.Compact.DefaultCount),
 		redact.Safe(m.Compact.DeleteOnlyCount),
 		redact.Safe(m.Compact.ElisionOnlyCount),
 		redact.Safe(m.Compact.MoveCount),
 		redact.Safe(m.Compact.ReadCount),
 		redact.Safe(m.Compact.RewriteCount),
+		redact.Safe(m.Compact.CopyCount),
 		redact.Safe(m.Compact.MultiLevelCount))
 
 	w.Printf("MemTables: %d (%s)  zombie: %d (%s)\n",
diff --git a/objstorage/objstorageprovider/readahead.go b/objstorage/objstorageprovider/readahead.go
index e07017c9..e29155d8 100644
--- a/objstorage/objstorageprovider/readahead.go
+++ b/objstorage/objstorageprovider/readahead.go
@@ -40,40 +40,7 @@ func makeReadaheadState(maxReadaheadSize int64) readaheadState {
 }
 
 func (rs *readaheadState) recordCacheHit(offset, blockLength int64) {
-	currentReadEnd := offset + blockLength
-	if rs.numReads >= minFileReadsForReadahead {
-		if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize {
-			// This is a read that would have resulted in a readahead, had it
-			// not been a cache hit.
-			rs.limit = currentReadEnd
-			return
-		}
-		if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+rs.maxReadaheadSize {
-			// We read too far away from rs.limit to benefit from readahead in
-			// any scenario. Reset all variables.
-			rs.numReads = 1
-			rs.limit = currentReadEnd
-			rs.size = initialReadaheadSize
-			rs.prevSize = 0
-			return
-		}
-		// Reads in the range [rs.limit - rs.prevSize, rs.limit] end up
-		// here. This is a read that is potentially benefitting from a past
-		// readahead.
-		return
-	}
-	if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize {
-		// Blocks are being read sequentially and would benefit from readahead
-		// down the line.
-		rs.numReads++
-		return
-	}
-	// We read too far ahead of the last read, or before it. This indicates
-	// a random read, where readahead is not desirable. Reset all variables.
-	rs.numReads = 1
-	rs.limit = currentReadEnd
-	rs.size = initialReadaheadSize
-	rs.prevSize = 0
+	_ = rs.maybeReadaheadOrCacheHit(offset, blockLength, false)
 }
 
 // maybeReadahead updates state and determines whether to issue a readahead /
@@ -81,6 +48,13 @@ func (rs *readaheadState) recordCacheHit(offset, blockLength int64) {
 // Returns a size value (greater than 0) that should be prefetched if readahead
 // would be beneficial.
 func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
+	return rs.maybeReadaheadOrCacheHit(offset, blockLength, true)
+}
+
+// The return value should be ignored if !readahead.
+func (rs *readaheadState) maybeReadaheadOrCacheHit(
+	offset, blockLength int64, readahead bool,
+) int64 {
 	currentReadEnd := offset + blockLength
 	if rs.numReads >= minFileReadsForReadahead {
 		// The minimum threshold of sequential reads to justify reading ahead
@@ -119,18 +93,22 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
 			//
 			//
 			rs.numReads++
-			rs.limit = offset + rs.size
-			rs.prevSize = rs.size
-			// Increase rs.size for the next read.
-			rs.size *= 2
-			if rs.size > rs.maxReadaheadSize {
-				rs.size = rs.maxReadaheadSize
+			if readahead {
+				rs.limit = offset + rs.size
+				rs.prevSize = rs.size
+				// Increase rs.size for the next read.
+				rs.size *= 2
+				if rs.size > rs.maxReadaheadSize {
+					rs.size = rs.maxReadaheadSize
+				}
+			} else {
+				// This is a read that would have resulted in a readahead, had it
+				// not been a cache hit.
+				rs.limit = currentReadEnd
 			}
 			return rs.prevSize
 		}
 		if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+rs.maxReadaheadSize {
-			// The above conditional has rs.limit > rs.prevSize to confirm that
-			// rs.limit - rs.prevSize would not underflow.
 			// We read too far away from rs.limit to benefit from readahead in
 			// any scenario. Reset all variables.
 			// The case where we read too far ahead:
@@ -156,7 +134,16 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
 
 			return 0
 		}
-		// Reads in the range [rs.limit - rs.prevSize, rs.limit] end up
+		// The previous if-block predicates were all false. This mechanically implies:
+		//
+		// INVARIANT:
+		//  !(currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize) &&
+		//  !(currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+rs.maxReadaheadSize)
+		// Which mechanically simplifies to:
+		//  currentReadEnd < rs.limit  && currentReadEnd >= rs.limit-rs.prevSize &&
+		//  offset <= rs.limit+rs.maxReadaheadSize
+		//
+		// So reads in the range [rs.limit - rs.prevSize, rs.limit] end up
 		// here. This is a read that is potentially benefitting from a past
 		// readahead, but there's no reason to issue a readahead call at the
 		// moment.
@@ -171,6 +158,8 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
 		rs.numReads++
 		return 0
 	}
+	// Not yet at the numReads threshold to justify readahead. But we want to
+	// capture whether readahead will be beneficial in the future.
 	if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize {
 		// Blocks are being read sequentially and would benefit from readahead
 		// down the line.
@@ -182,6 +171,16 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
 		//                offset    currentReadEnd
 		//
 		rs.numReads++
+		// It is possible to fall here when rs.limit has not been initialized. If
+		// we don't initialize, rs.limit, it is possible that the first read
+		// offset was at rs.limit+rs.maxReadaheadSize-delta and the enclosing
+		// if-block predicate was true, and the next read is sequential but has
+		// offset > rs.limit+rs.maxReadaheadSize (if we left rs.limit at 0), and
+		// the enclosing if-block predicate will be false and we will incorrectly
+		// think that readahead is not beneficial. The same issue arises if
+		// rs.limit has been initialized and currentReadEnd is advancing past
+		// rs.limit.
+		rs.limit = currentReadEnd
 		return 0
 	}
 	// We read too far ahead of the last read, or before it. This indicates
diff --git a/options.go b/options.go
index 20166612..545d561e 100644
--- a/options.go
+++ b/options.go
@@ -682,10 +682,6 @@ type Options struct {
 		// major version is at least `FormatFlushableIngest`.
 		DisableIngestAsFlushable func() bool
 
-		// CheckExternalIngestions enables opening external ssts at ingest time and
-		// validating that they are not empty. Used for testing/debugging.
-		CheckExternalIngestions bool
-
 		// RemoteStorage enables use of remote storage (e.g. S3) for storing
 		// sstables. Setting this option enables use of CreateOnShared option and
 		// allows ingestion of external files.
diff --git a/replay/replay.go b/replay/replay.go
index b6f6034f..4040a7d6 100644
--- a/replay/replay.go
+++ b/replay/replay.go
@@ -104,6 +104,7 @@ type Metrics struct {
 		Move        int64
 		Read        int64
 		Rewrite     int64
+		Copy        int64
 		MultiLevel  int64
 	}
 	EstimatedDebt SampledMetric
@@ -169,6 +170,7 @@ func (m *Metrics) WriteBenchmarkString(name string, w io.Writer) error {
 			{Value: float64(m.CompactionCounts.Move), Unit: "move"},
 			{Value: float64(m.CompactionCounts.Read), Unit: "read"},
 			{Value: float64(m.CompactionCounts.Rewrite), Unit: "rewrite"},
+			{Value: float64(m.CompactionCounts.Copy), Unit: "copy"},
 			{Value: float64(m.CompactionCounts.MultiLevel), Unit: "multilevel"},
 		}},
 		// Total database sizes sampled after every workload step and
@@ -555,6 +557,7 @@ func (r *Runner) Wait() (Metrics, error) {
 	m.CompactionCounts.Move = pm.Compact.MoveCount
 	m.CompactionCounts.Read = pm.Compact.ReadCount
 	m.CompactionCounts.Rewrite = pm.Compact.RewriteCount
+	m.CompactionCounts.Copy = pm.Compact.CopyCount
 	m.CompactionCounts.MultiLevel = pm.Compact.MultiLevelCount
 	m.Ingest.BytesIntoL0 = pm.Levels[0].BytesIngested
 	m.Ingest.BytesWeightedByLevel = ingestBytesWeighted
diff --git a/sstable/copier.go b/sstable/copier.go
index 96fe3cb8..207b427f 100644
--- a/sstable/copier.go
+++ b/sstable/copier.go
@@ -31,7 +31,10 @@ import (
 //
 // The resulting sstable will have no block properties.
 //
-// Closes input and finishes or aborts output, including on non-nil errors.
+// The function might return ErrEmptySpan if there are no blocks that could
+// include keys in the given range. See ErrEmptySpan for more details.
+//
+// Closes input and finishes or aborts output in all cases, including on errors.
 //
 // Note that CopySpan is not aware of any suffix or prefix replacement; the
 // caller must account for those when specifying the bounds.
@@ -81,7 +84,8 @@ func CopySpan(
 	}()
 
 	if r.Properties.NumValueBlocks > 0 || r.Properties.NumRangeKeys() > 0 || r.Properties.NumRangeDeletions > 0 {
-		return 0, errors.New("cannot CopySpan sstables with value blocks or range keys")
+		// We just checked for these conditions above.
+		return 0, base.AssertionFailedf("cannot CopySpan sstables with value blocks or range keys")
 	}
 
 	// Set the filter block to be copied over if it exists. It will return false
@@ -128,7 +132,7 @@ func CopySpan(
 	// a non-empty sst by copying something outside the span, but #3907 means that
 	// the empty virtual span would still be a problem, so don't bother.
 	if len(blocks) < 1 {
-		return 0, errors.Newf("CopySpan cannot copy empty span %s %s", start, end)
+		return 0, ErrEmptySpan
 	}
 
 	// Find the span of the input file that contains all our blocks, and then copy
@@ -171,6 +175,15 @@ func CopySpan(
 	return wrote, nil
 }
 
+// ErrEmptySpan is returned by CopySpan if the input sstable has no keys in the
+// requested span.
+//
+// Note that CopySpan's determination of block overlap is best effort - we may
+// copy a block that doesn't actually contain any keys in the span, in which
+// case we won't generate this error. We currently only generate this error when
+// the span start is beyond all keys in the physical sstable.
+var ErrEmptySpan = errors.New("cannot copy empty span")
+
 // indexEntry captures the two components of an sst index entry: the key and the
 // decoded block handle value.
 type indexEntry struct {
diff --git a/version_set.go b/version_set.go
index b3f17198..5f7130ed 100644
--- a/version_set.go
+++ b/version_set.go
@@ -845,6 +845,15 @@ func (vs *versionSet) incrementCompactions(
 	case compactionKindRewrite:
 		vs.metrics.Compact.Count++
 		vs.metrics.Compact.RewriteCount++
+
+	case compactionKindCopy:
+		vs.metrics.Compact.Count++
+		vs.metrics.Compact.CopyCount++
+
+	default:
+		if invariants.Enabled {
+			panic("unhandled compaction kind")
+		}
 	}
 	if len(extraLevels) > 0 {
 		vs.metrics.Compact.MultiLevelCount++


Test patch:
diff --git a/data_test.go b/data_test.go
index 3ee315c1..5ea1ba66 100644
--- a/data_test.go
+++ b/data_test.go
@@ -842,6 +842,16 @@ func runDBDefineCmdReuseFS(td *datadriven.TestData, opts *Options) (*DB, error)
 			for _, levelOpts := range opts.Levels {
 				levelOpts.BlockSize = size
 			}
+		case "bloom-bits-per-key":
+			v, err := strconv.Atoi(arg.Vals[0])
+			if err != nil {
+				return nil, err
+			}
+			fp := bloom.FilterPolicy(v)
+			opts.Filters = map[string]FilterPolicy{fp.Name(): fp}
+			for i := range opts.Levels {
+				opts.Levels[i].FilterPolicy = fp
+			}
 		case "format-major-version":
 			fmv, err := strconv.Atoi(arg.Vals[0])
 			if err != nil {
diff --git a/metrics_test.go b/metrics_test.go
index d1cec478..f0db9587 100644
--- a/metrics_test.go
+++ b/metrics_test.go
@@ -40,7 +40,8 @@ func exampleMetrics() Metrics {
 	m.Compact.MoveCount = 30
 	m.Compact.ReadCount = 31
 	m.Compact.RewriteCount = 32
-	m.Compact.MultiLevelCount = 33
+	m.Compact.CopyCount = 33
+	m.Compact.MultiLevelCount = 34
 	m.Compact.EstimatedDebt = 6
 	m.Compact.InProgressBytes = 7
 	m.Compact.NumInProgress = 2
diff --git a/objstorage/objstorageprovider/readahead_test.go b/objstorage/objstorageprovider/readahead_test.go
index a44d78ac..86aff7b5 100644
--- a/objstorage/objstorageprovider/readahead_test.go
+++ b/objstorage/objstorageprovider/readahead_test.go
@@ -20,9 +20,7 @@ func TestMaybeReadahead(t *testing.T) {
 		cacheHit := false
 		switch d.Cmd {
 		case "reset":
-			rs.size = initialReadaheadSize
-			rs.limit = 0
-			rs.numReads = 0
+			rs = makeReadaheadState(rs.maxReadaheadSize)
 			return ""
 
 		case "cache-read":
diff --git a/objstorage/objstorageprovider/testdata/readahead b/objstorage/objstorageprovider/testdata/readahead
index d2bb2173..adfe4ec6 100644
--- a/objstorage/objstorageprovider/testdata/readahead
+++ b/objstorage/objstorageprovider/testdata/readahead
@@ -8,7 +8,7 @@ readahead:  0
 numReads:   1
 size:       65536
 prevSize:   0
-limit:      0
+limit:      2064
 
 read
 2096, 16
@@ -17,7 +17,7 @@ readahead:  0
 numReads:   2
 size:       65536
 prevSize:   0
-limit:      0
+limit:      2112
 
 read
 2112, 16
@@ -84,7 +84,7 @@ readahead:  0
 numReads:   2
 size:       65536
 prevSize:   0
-limit:      16208
+limit:      16209
 
 # The next read is too far ahead to benefit from readahead
 # (i.e. 540497 > 16208 (limit) + (512 << 10) (maxReadaheadSize))
@@ -126,45 +126,49 @@ readahead:  0
 numReads:   2
 size:       65536
 prevSize:   0
-limit:      16
+limit:      7796
 
 read
-7680, 16
+7800, 16
 ----
 readahead:  65536
 numReads:   3
 size:       131072
 prevSize:   65536
-limit:      73216
+limit:      73336
 
 read
-7780, 16
----
+7880, 16
+----
 readahead:  0
 numReads:   4
 size:       131072
 prevSize:   65536
-limit:      73216
+limit:      73336
 
 read
-7880, 16
+7980, 16
 ----
-expected 2 args: offset, size
+readahead:  0
+numReads:   5
+size:       131072
+prevSize:   65536
+limit:      73336
 
 read
-7980, 16
+8080, 16
 ----
 readahead:  0
-numReads:   4
+numReads:   6
 size:       131072
 prevSize:   65536
-limit:      73216
+limit:      73336
 
 read
 73416, 16
 ----
 readahead:  131072
-numReads:   5
+numReads:   7
 size:       262144
 prevSize:   131072
 limit:      204488
@@ -173,7 +177,7 @@ read
 204488, 16
 ----
 readahead:  262144
-numReads:   6
+numReads:   8
 size:       262144
 prevSize:   262144
 limit:      466632
@@ -184,7 +188,7 @@ read
 466632, 16
 ----
 readahead:  262144
-numReads:   7
+numReads:   9
 size:       262144
 prevSize:   262144
 limit:      728776
@@ -195,7 +199,7 @@ cache-read
 728770, 16
 ----
 readahead:  0
-numReads:   7
+numReads:   10
 size:       262144
 prevSize:   262144
 limit:      728786
@@ -204,7 +208,7 @@ read
 728780, 16
 ----
 readahead:  262144
-numReads:   8
+numReads:   11
 size:       262144
 prevSize:   262144
 limit:      990924
@@ -219,3 +223,36 @@ numReads:   1
 size:       65536
 prevSize:   0
 limit:      1216
+
+reset
+----
+
+# Offset 240KiB, length 32KiB.
+read
+245760, 32768
+----
+readahead:  0
+numReads:   1
+size:       65536
+prevSize:   0
+limit:      278528
+
+# Offset 280KiB, length 32KiB.
+read
+286720, 32768
+----
+readahead:  0
+numReads:   2
+size:       65536
+prevSize:   0
+limit:      319488
+
+# Offset 320KiB, length 32KiB.
+read
+327680, 32768
+----
+readahead:  65536
+numReads:   3
+size:       131072
+prevSize:   65536
+limit:      393216
diff --git a/replay/replay_test.go b/replay/replay_test.go
index 2c3b7eb5..516092dc 100644
--- a/replay/replay_test.go
+++ b/replay/replay_test.go
@@ -441,7 +441,7 @@ func TestBenchmarkString(t *testing.T) {
 	var buf bytes.Buffer
 	require.NoError(t, m.WriteBenchmarkString("tpcc", &buf))
 	require.Equal(t, strings.TrimSpace(`
-BenchmarkBenchmarkReplay/tpcc/CompactionCounts 1 0 compactions 0 default 0 delete 0 elision 0 move 0 read 0 rewrite 0 multilevel
+BenchmarkBenchmarkReplay/tpcc/CompactionCounts 1 0 compactions 0 default 0 delete 0 elision 0 move 0 read 0 rewrite 0 copy 0 multilevel
 BenchmarkBenchmarkReplay/tpcc/DatabaseSize/mean 1 5.36870912e+09 bytes
 BenchmarkBenchmarkReplay/tpcc/DatabaseSize/max 1 5.36870912e+09 bytes
 BenchmarkBenchmarkReplay/tpcc/DurationWorkload 1 1 sec/op
diff --git a/testdata/event_listener b/testdata/event_listener
index 08537b56..8db0b6d7 100644
--- a/testdata/event_listener
+++ b/testdata/event_listener
@@ -222,7 +222,7 @@ total |     3  1.7KB     0B       0 |     - |  671B |     1   590B |     0     0
 WAL: 1 files (0B)  in: 48B  written: 81B (69% overhead)
 Flushes: 3
 Compactions: 1  estimated debt: 1.7KB  in progress: 0 (0B)
-             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -322,7 +322,7 @@ total |     6  3.5KB     0B       0 |     - | 1.8KB |     3  1.7KB |     0     0
 WAL: 1 files (0B)  in: 82B  written: 108B (32% overhead)
 Flushes: 6
 Compactions: 1  estimated debt: 3.5KB  in progress: 0 (0B)
-             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (512KB)  zombie: 1 (512KB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
diff --git a/testdata/ingest b/testdata/ingest
index c27b75fc..fc6a64bf 100644
--- a/testdata/ingest
+++ b/testdata/ingest
@@ -46,7 +46,7 @@ total |     1   569B     0B       0 |     - |  569B |     1   569B |     0     0
 WAL: 1 files (0B)  in: 0B  written: 0B (0% overhead)
 Flushes: 0
 Compactions: 0  estimated debt: 0B  in progress: 0 (0B)
-             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 0 (0B)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
diff --git a/testdata/ingest_external b/testdata/ingest_external
index 71921b1c..dc6edf5a 100644
--- a/testdata/ingest_external
+++ b/testdata/ingest_external
@@ -830,3 +830,81 @@ a: (foo, .)
 aa@10: (a, .)
 b@11: (b, .)
 .
+
+# Test empty external ingestion.
+reset
+----
+
+build-remote ext
+set a@10 a
+set b@11 b
+----
+
+ingest-external
+ext bounds=(u,v)
+----
+
+lsm
+----
+L6:
+  000004(000004):[u#10,DELSIZED-v#inf,RANGEDEL]
+
+iter
+first
+----
+.
+
+download a z via-backing-file-download
+----
+ok
+
+# LSM should be empty now.
+lsm
+----
+
+iter
+first
+----
+.
+
+# Test with two external files, one empty.
+ingest-external
+ext bounds=(a,c)
+----
+
+ingest-external
+ext bounds=(u,v)
+----
+
+lsm
+----
+L6:
+  000006(000006):[a#11,DELSIZED-c#inf,RANGEDEL]
+  000007(000006):[u#12,DELSIZED-v#inf,RANGEDEL]
+
+iter
+first
+next
+next
+----
+a@10: (a, .)
+b@11: (b, .)
+.
+
+download a z
+----
+ok
+
+lsm
+----
+L6:
+  000008:[a@10#0,SET-b@11#0,SET]
+
+iter
+first
+next
+next
+----
+a@10: (a, .)
+b@11: (b, .)
+.
diff --git a/testdata/iter_histories/prefix_iteration b/testdata/iter_histories/prefix_iteration
index 1ea878ea..d977b2ec 100644
--- a/testdata/iter_histories/prefix_iteration
+++ b/testdata/iter_histories/prefix_iteration
@@ -336,3 +336,29 @@ seek-prefix-ge c@9
 ----
 .
 .
+
+# Regression test for #3610.
+#
+# Similar to the above case, this test consists of two SeekPrefixGEs for
+# ascending keys, resulting in TrySeekUsingNext()=true for the second seek.
+# Previously, during the first SeekPrefixGE the mergingIter could Next the
+# levelIter beyond the file containing point keys relevant to both seeks.
+
+define bloom-bits-per-key=100
+L4
+  b@0.SET.10:b@0
+L5
+  b@8.RANGEDEL.3:b@1
+  c@3.SET.0:c@3
+----
+L4:
+  000004:[b@0#10,SET-b@0#10,SET]
+L5:
+  000005:[b@8#3,RANGEDEL-c@3#0,SET]
+
+combined-iter
+seek-prefix-ge b@10
+seek-prefix-ge c@10
+----
+b@0: (b@0, .)
+c@3: (c@3, .)
diff --git a/testdata/metrics b/testdata/metrics
index 91470066..04e93ab5 100644
--- a/testdata/metrics
+++ b/testdata/metrics
@@ -15,7 +15,7 @@ total |  2.8K  2.7KB     0B    2.8K |     - | 2.8KB |  2.9K  2.8KB |  2.9K  2.8K
 WAL: 22 files (24B)  in: 25B  written: 26B (4% overhead)
 Flushes: 8
 Compactions: 5  estimated debt: 6B  in progress: 2 (7B)
-             default: 27  delete: 28  elision: 29  move: 30  read: 31  rewrite: 32  multi-level: 33
+             default: 27  delete: 28  elision: 29  move: 30  read: 31  rewrite: 32  copy: 33  multi-level: 34
 MemTables: 12 (11B)  zombie: 14 (13B)
 Zombie tables: 16 (15B, local: 30B)
 Backing tables: 1 (2.0MB)
@@ -66,7 +66,7 @@ total |     1   589B     0B       0 |     - |   28B |     0     0B |     0     0
 WAL: 1 files (0B)  in: 17B  written: 28B (65% overhead)
 Flushes: 1
 Compactions: 0  estimated debt: 0B  in progress: 0 (0B)
-             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -121,7 +121,7 @@ total |     1   595B     0B       0 |     - |   56B |     0     0B |     0     0
 WAL: 1 files (0B)  in: 34B  written: 56B (65% overhead)
 Flushes: 2
 Compactions: 1  estimated debt: 0B  in progress: 0 (0B)
-             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 2 (512KB)
 Zombie tables: 2 (1.2KB, local: 1.2KB)
 Backing tables: 0 (0B)
@@ -163,7 +163,7 @@ total |     1   595B     0B       0 |     - |   56B |     0     0B |     0     0
 WAL: 1 files (0B)  in: 34B  written: 56B (65% overhead)
 Flushes: 2
 Compactions: 1  estimated debt: 0B  in progress: 0 (0B)
-             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 2 (512KB)
 Zombie tables: 2 (1.2KB, local: 1.2KB)
 Backing tables: 0 (0B)
@@ -202,7 +202,7 @@ total |     1   595B     0B       0 |     - |   56B |     0     0B |     0     0
 WAL: 1 files (0B)  in: 34B  written: 56B (65% overhead)
 Flushes: 2
 Compactions: 1  estimated debt: 0B  in progress: 0 (0B)
-             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 2 (512KB)
 Zombie tables: 1 (589B, local: 589B)
 Backing tables: 0 (0B)
@@ -245,7 +245,7 @@ total |     1   595B     0B       0 |     - |   56B |     0     0B |     0     0
 WAL: 1 files (0B)  in: 34B  written: 56B (65% overhead)
 Flushes: 2
 Compactions: 1  estimated debt: 0B  in progress: 0 (0B)
-             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -315,7 +315,7 @@ total |     4  2.6KB    38B       0 |     - |  149B |     0     0B |     0     0
 WAL: 1 files (0B)  in: 116B  written: 149B (28% overhead)
 Flushes: 3
 Compactions: 1  estimated debt: 2.6KB  in progress: 0 (0B)
-             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -369,7 +369,7 @@ total |     3  2.0KB    41B       0 |     - |  149B |     0     0B |     0     0
 WAL: 1 files (0B)  in: 116B  written: 149B (28% overhead)
 Flushes: 3
 Compactions: 2  estimated debt: 0B  in progress: 0 (0B)
-             default: 2  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 2  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -472,7 +472,7 @@ total |     7  4.3KB    41B       0 |     - | 1.9KB |     3  1.7KB |     0     0
 WAL: 1 files (0B)  in: 176B  written: 187B (6% overhead)
 Flushes: 8
 Compactions: 2  estimated debt: 4.3KB  in progress: 0 (0B)
-             default: 2  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 2  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (1.0MB)  zombie: 1 (1.0MB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -534,7 +534,7 @@ total |    10  6.1KB    41B       0 |     - | 2.0KB |     3  1.7KB |     0     0
 WAL: 1 files (0B)  in: 223B  written: 245B (10% overhead)
 Flushes: 9
 Compactions: 2  estimated debt: 6.1KB  in progress: 0 (0B)
-             default: 2  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 2  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (1.0MB)  zombie: 1 (1.0MB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -610,7 +610,7 @@ total |    11  5.6KB    41B       2 |     - | 2.5KB |     4  2.3KB |     0     0
 WAL: 1 files (0B)  in: 223B  written: 245B (10% overhead)
 Flushes: 9
 Compactions: 2  estimated debt: 5.6KB  in progress: 0 (0B)
-             default: 2  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 2  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (1.0MB)  zombie: 1 (1.0MB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 2 (1.2KB)
@@ -711,7 +711,7 @@ total |     6  3.8KB    41B       0 |     - | 3.1KB |     5  2.9KB |     0     0
 WAL: 1 files (0B)  in: 223B  written: 245B (10% overhead)
 Flushes: 9
 Compactions: 3  estimated debt: 0B  in progress: 0 (0B)
-             default: 3  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 3  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (1.0MB)  zombie: 1 (1.0MB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -764,7 +764,7 @@ total |     1   604B     0B       0 |     - |   38B |     0     0B |     0     0
 WAL: 1 files (0B)  in: 27B  written: 38B (41% overhead)
 Flushes: 1
 Compactions: 0  estimated debt: 0B  in progress: 0 (0B)
-             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -800,8 +800,8 @@ total |     1   604B     0B       0 |     - |   38B |     0     0B |     0     0
 -------------------------------------------------------------------------------------------------------------------
 WAL: 1 files (0B)  in: 27B  written: 38B (41% overhead)
 Flushes: 1
-Compactions: 0  estimated debt: 0B  in progress: 0 (0B)
-             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+Compactions: 1  estimated debt: 0B  in progress: 0 (0B)
+             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 1  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -846,8 +846,8 @@ total |     2  1.2KB     0B       0 |     - |  627B |     1   589B |     0     0
 -------------------------------------------------------------------------------------------------------------------
 WAL: 1 files (0B)  in: 27B  written: 38B (41% overhead)
 Flushes: 1
-Compactions: 0  estimated debt: 1.2KB  in progress: 0 (0B)
-             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+Compactions: 1  estimated debt: 1.2KB  in progress: 0 (0B)
+             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 1  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -893,8 +893,8 @@ total |     3  1.7KB     0B       0 |     - |  655B |     1   589B |     0     0
 -------------------------------------------------------------------------------------------------------------------
 WAL: 1 files (0B)  in: 44B  written: 66B (50% overhead)
 Flushes: 2
-Compactions: 0  estimated debt: 1.7KB  in progress: 0 (0B)
-             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+Compactions: 1  estimated debt: 1.7KB  in progress: 0 (0B)
+             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 1  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -932,7 +932,7 @@ total |     3  1.7KB     0B       0 |     - |    0B |     0     0B |     0     0
 WAL: 1 files (0B)  in: 0B  written: 0B (0% overhead)
 Flushes: 0
 Compactions: 0  estimated debt: 1.7KB  in progress: 0 (0B)
-             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 0 (0B)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
@@ -969,7 +969,7 @@ total |     1   603B     0B       0 |     - |    0B |     0     0B |     0     0
 WAL: 1 files (0B)  in: 0B  written: 0B (0% overhead)
 Flushes: 0
 Compactions: 1  estimated debt: 0B  in progress: 0 (0B)
-             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 0 (0B)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
diff --git a/tool/testdata/db_lsm b/tool/testdata/db_lsm
index 7dff233b..d6e45965 100644
--- a/tool/testdata/db_lsm
+++ b/tool/testdata/db_lsm
@@ -25,7 +25,7 @@ total |     1   709B     0B       0 |     - |    0B |     0     0B |     0     0
 WAL: 0 files (0B)  in: 0B  written: 0B (0% overhead)
 Flushes: 0
 Compactions: 0  estimated debt: 0B  in progress: 0 (0B)
-             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
+             default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 0 (0B)
 Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)