Repository: cockroachdb/pebble
Pull request #3682 (2024-06-21 17:58:20)

*: add facility to print the iterator stack
To print the entire iterator stack tree, we can now use
`fmt.Printf("%s", base.DebugTree(iter))`
to get something like this:
```
*pebble.Iterator(0x14003a0ea08)
 ├── *keyspan.InterleavingIter(0x14003ee2030)
 │    ├── *invalidating.iter(0x14007c52f00)
 │    │    └── *pebble.mergingIter(0x14003a0f028)
 │    │         └── *pebble.levelIter(0x14003a0f3a0) L6: fileNum=001316
 │    │              └── *sstable.singleLevelIterator(0x14004bce008) fileNum=001316
 │    └── *keyspan.assertIter(0x14001229950)
 │         └── *keyspan.DefragmentingIter(0x14003ee26e8)
 │              └── *keyspan.BoundedIter(0x14003ee2678)
 │                   └── *keyspanimpl.MergingIter(0x14003ee2300)
 │                        └── *keyspanimpl.LevelIter(0x14003ee2788) L6
 │                             └── *keyspan.assertIter(0x14000753a70)
 │                                  └── *sstable.fragmentBlockIter(0x140000ffb88) fileNum=001316
 └── *invalidating.iter(0x14007c52f00)
      └── *pebble.mergingIter(0x14003a0f028)
           └── *pebble.levelIter(0x14003a0f3a0) L6: fileNum=001316
                └── *sstable.singleLevelIterator(0x14004bce008) fileNum=001316
```

cockroach-teamcity: 

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/cockroachdb/pebble/3682)
<!-- Reviewable:end -->

nicktrav: This is really nice! A couple comments. 

Any way to annotate the level iters with level numbers? There's possibly other metadata that might be useful too? Position? Direction? Data iterated over so far, etc?

Could this then be useful up in Cockroach, say, for trace spans - we can print this debug stack. It could supplement the iterator stats nicely.
RaduBerinde: Ok to merge? This has been useful when debugging some meta failures.
RaduBerinde: Friendly ping
RaduBerinde: TFTR!

Patch:
diff --git a/batch.go b/batch.go
index c9c3efe5..65b0fe3c 100644
--- a/batch.go
+++ b/batch.go
@@ -27,6 +27,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/rangedel"
 	"github.com/cockroachdb/pebble/internal/rangekey"
 	"github.com/cockroachdb/pebble/internal/rawalloc"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 )
 
 const (
@@ -1827,6 +1828,11 @@ func (i *batchIter) SetBounds(lower, upper []byte) {
 
 func (i *batchIter) SetContext(_ context.Context) {}
 
+// DebugTree is part of the InternalIterator interface.
+func (i *batchIter) DebugTree(tp treeprinter.Node) {
+	tp.Childf("%T(%p)", i, i)
+}
+
 type flushableBatchEntry struct {
 	// offset is the byte offset of the record within the batch repr.
 	offset uint32
@@ -2323,6 +2329,11 @@ func (i *flushableBatchIter) SetBounds(lower, upper []byte) {
 
 func (i *flushableBatchIter) SetContext(_ context.Context) {}
 
+// DebugTree is part of the InternalIterator interface.
+func (i *flushableBatchIter) DebugTree(tp treeprinter.Node) {
+	tp.Childf("%T(%p)", i, i)
+}
+
 // flushFlushableBatchIter is similar to flushableBatchIter but it keeps track
 // of number of bytes iterated.
 type flushFlushableBatchIter struct {
diff --git a/error_iter.go b/error_iter.go
index b9f4c5ba..61911c9f 100644
--- a/error_iter.go
+++ b/error_iter.go
@@ -9,6 +9,7 @@ import (
 
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/keyspan"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 )
 
 type errorIter struct {
@@ -72,6 +73,10 @@ func (c *errorIter) SetBounds(lower, upper []byte) {}
 
 func (c *errorIter) SetContext(_ context.Context) {}
 
+func (c *errorIter) DebugTree(tp treeprinter.Node) {
+	tp.Childf("%T(%p)", c, c)
+}
+
 type errorKeyspanIter struct {
 	err error
 }
@@ -88,3 +93,4 @@ func (i *errorKeyspanIter) Prev() (*keyspan.Span, error)             { return ni
 func (i *errorKeyspanIter) Close()                                   {}
 func (*errorKeyspanIter) String() string                             { return "error" }
 func (*errorKeyspanIter) WrapChildren(wrap keyspan.WrapFn)           {}
+func (i *errorKeyspanIter) DebugTree(tp treeprinter.Node)            { tp.Childf("%T(%p)", i, i) }
diff --git a/get_iter.go b/get_iter.go
index f9c1b965..511685b0 100644
--- a/get_iter.go
+++ b/get_iter.go
@@ -11,6 +11,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/manifest"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 )
 
 // getIter is an internal iterator used to perform gets. It iterates through
@@ -161,6 +162,14 @@ func (g *getIter) SetBounds(lower, upper []byte) {
 
 func (g *getIter) SetContext(_ context.Context) {}
 
+// DebugTree is part of the InternalIterator interface.
+func (g *getIter) DebugTree(tp treeprinter.Node) {
+	n := tp.Childf("%T(%p)", g, g)
+	if g.iter != nil {
+		g.iter.DebugTree(n)
+	}
+}
+
 func (g *getIter) initializeNextIterator() (ok bool) {
 	// A batch's keys shadow all other keys, so we visit the batch first.
 	if g.batch != nil {
diff --git a/internal/arenaskl/iterator.go b/internal/arenaskl/iterator.go
index d23e4168..21176c0e 100644
--- a/internal/arenaskl/iterator.go
+++ b/internal/arenaskl/iterator.go
@@ -22,6 +22,7 @@ import (
 	"sync"
 
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 )
 
 type splice struct {
@@ -248,6 +249,11 @@ func (it *Iterator) SetBounds(lower, upper []byte) {
 // SetContext implements base.InternalIterator.
 func (it *Iterator) SetContext(_ context.Context) {}
 
+// DebugTree is part of the InternalIterator interface.
+func (it *Iterator) DebugTree(tp treeprinter.Node) {
+	tp.Childf("%T(%p)", it, it)
+}
+
 func (it *Iterator) decodeKey() {
 	it.kv.K.UserKey = it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize)
 	it.kv.K.Trailer = it.nd.keyTrailer
diff --git a/internal/base/iterator.go b/internal/base/iterator.go
index 17b3b1b3..3c204144 100644
--- a/internal/base/iterator.go
+++ b/internal/base/iterator.go
@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/cockroachdb/pebble/internal/humanize"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 	"github.com/cockroachdb/redact"
 )
 
@@ -217,6 +218,8 @@ type InternalIterator interface {
 	SetContext(ctx context.Context)
 
 	fmt.Stringer
+
+	IteratorDebug
 }
 
 // TopLevelIterator extends InternalIterator to include an additional absolute
@@ -461,3 +464,19 @@ func (s *InternalIteratorStats) SafeFormat(p redact.SafePrinter, verb rune) {
 			humanize.Bytes.Uint64(s.SeparatedPointValue.ValueBytesFetched))
 	}
 }
+
+// IteratorDebug is an interface implemented by all internal iterators and
+// fragment iterators.
+type IteratorDebug interface {
+	// DebugTree prints the entire iterator stack, used for debugging.
+	//
+	// Each implementation should perform a single Child/Childf call on tp.
+	DebugTree(tp treeprinter.Node)
+}
+
+// DebugTree returns the iterator tree as a multi-line string.
+func DebugTree(iter IteratorDebug) string {
+	tp := treeprinter.New()
+	iter.DebugTree(tp)
+	return tp.String()
+}
diff --git a/internal/invalidating/iter.go b/internal/invalidating/iter.go
index aa6e244f..82b50276 100644
--- a/internal/invalidating/iter.go
+++ b/internal/invalidating/iter.go
@@ -9,6 +9,7 @@ import (
 
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 )
 
 // MaybeWrapIfInvariants wraps some iterators with an invalidating iterator.
@@ -159,6 +160,14 @@ func (i *iter) SetContext(ctx context.Context) {
 	i.iter.SetContext(ctx)
 }
 
+// DebugTree is part of the InternalIterator interface.
+func (i *iter) DebugTree(tp treeprinter.Node) {
+	n := tp.Childf("%T(%p)", i, i)
+	if i.iter != nil {
+		i.iter.DebugTree(n)
+	}
+}
+
 func (i *iter) String() string {
 	return i.iter.String()
 }
diff --git a/internal/keyspan/assert_iter.go b/internal/keyspan/assert_iter.go
index 98746e4d..42f76bb9 100644
--- a/internal/keyspan/assert_iter.go
+++ b/internal/keyspan/assert_iter.go
@@ -10,6 +10,7 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 )
 
 // Assert wraps an iterator which asserts that operations return sane results.
@@ -173,3 +174,11 @@ func (i *assertIter) Close() {
 func (i *assertIter) WrapChildren(wrap WrapFn) {
 	i.iter = wrap(i.iter)
 }
+
+// DebugTree is part of the FragmentIterator interface.
+func (i *assertIter) DebugTree(tp treeprinter.Node) {
+	n := tp.Childf("%T(%p)", i, i)
+	if i.iter != nil {
+		i.iter.DebugTree(n)
+	}
+}
diff --git a/internal/keyspan/bounded.go b/internal/keyspan/bounded.go
index 6b0e88a2..580cb74e 100644
--- a/internal/keyspan/bounded.go
+++ b/internal/keyspan/bounded.go
@@ -4,7 +4,10 @@
 
 package keyspan
 
-import "github.com/cockroachdb/pebble/internal/base"
+import (
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
+)
 
 // TODO(jackson): Consider removing this type and adding bounds enforcement
 // directly to the MergingIter. This type is probably too lightweight to warrant
@@ -266,3 +269,11 @@ func (i *BoundedIter) checkBackwardBound(span *Span, err error) (*Span, error) {
 func (i *BoundedIter) WrapChildren(wrap WrapFn) {
 	i.iter = wrap(i.iter)
 }
+
+// DebugTree is part of the FragmentIterator interface.
+func (i *BoundedIter) DebugTree(tp treeprinter.Node) {
+	n := tp.Childf("%T(%p)", i, i)
+	if i.iter != nil {
+		i.iter.DebugTree(n)
+	}
+}
diff --git a/internal/keyspan/defragment.go b/internal/keyspan/defragment.go
index 5f6402ce..eb9917d2 100644
--- a/internal/keyspan/defragment.go
+++ b/internal/keyspan/defragment.go
@@ -10,6 +10,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/bytealloc"
 	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 )
 
 // BufferReuseMaxCapacity is the maximum capacity of a DefragmentingIter buffer
@@ -558,3 +559,11 @@ func (i *DefragmentingIter) saveBytes(b []byte) []byte {
 func (i *DefragmentingIter) WrapChildren(wrap WrapFn) {
 	i.iter = wrap(i.iter)
 }
+
+// DebugTree is part of the FragmentIterator interface.
+func (i *DefragmentingIter) DebugTree(tp treeprinter.Node) {
+	n := tp.Childf("%T(%p)", i, i)
+	if i.iter != nil {
+		i.iter.DebugTree(n)
+	}
+}
diff --git a/internal/keyspan/filter.go b/internal/keyspan/filter.go
index 55a8d6f0..efba323d 100644
--- a/internal/keyspan/filter.go
+++ b/internal/keyspan/filter.go
@@ -4,7 +4,10 @@
 
 package keyspan
 
-import "github.com/cockroachdb/pebble/internal/base"
+import (
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
+)
 
 // FilterFunc is a callback that allows filtering keys from a Span. The result
 // is the set of keys that should be retained (using buf as a buffer). If the
@@ -130,3 +133,11 @@ func (i *filteringIter) filter(span *Span, dir int8) (*Span, error) {
 func (i *filteringIter) WrapChildren(wrap WrapFn) {
 	i.iter = wrap(i.iter)
 }
+
+// DebugTree is part of the FragmentIterator interface.
+func (i *filteringIter) DebugTree(tp treeprinter.Node) {
+	n := tp.Childf("%T(%p)", i, i)
+	if i.iter != nil {
+		i.iter.DebugTree(n)
+	}
+}
diff --git a/internal/keyspan/interleaving_iter.go b/internal/keyspan/interleaving_iter.go
index 41730c6c..5c02ed49 100644
--- a/internal/keyspan/interleaving_iter.go
+++ b/internal/keyspan/interleaving_iter.go
@@ -11,6 +11,7 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 )
 
 // A SpanMask may be used to configure an interleaving iterator to skip point
@@ -1141,6 +1142,17 @@ func (i *InterleavingIter) SetContext(ctx context.Context) {
 	i.pointIter.SetContext(ctx)
 }
 
+// DebugTree is part of the InternalIterator interface.
+func (i *InterleavingIter) DebugTree(tp treeprinter.Node) {
+	n := tp.Childf("%T(%p)", i, i)
+	if i.pointIter != nil {
+		i.pointIter.DebugTree(n)
+	}
+	if i.keyspanIter != nil {
+		i.keyspanIter.DebugTree(n)
+	}
+}
+
 // Invalidate invalidates the interleaving iterator's current position, clearing
 // its state. This prevents optimizations such as reusing the current span on
 // seek.
diff --git a/internal/keyspan/iter.go b/internal/keyspan/iter.go
index 217221c6..06ff261a 100644
--- a/internal/keyspan/iter.go
+++ b/internal/keyspan/iter.go
@@ -4,7 +4,10 @@
 
 package keyspan
 
-import "github.com/cockroachdb/pebble/internal/base"
+import (
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
+)
 
 // FragmentIterator defines an iterator interface over spans. The spans
 // surfaced by a FragmentIterator must be non-overlapping. This is achieved by
@@ -65,6 +68,8 @@ type FragmentIterator interface {
 	// function can call WrapChildren to recursively wrap an entire iterator
 	// stack. Used only for debug logging.
 	WrapChildren(wrap WrapFn)
+
+	base.IteratorDebug
 }
 
 // SpanIterOptions is a subset of IterOptions that are necessary to instantiate
@@ -210,8 +215,13 @@ func (i *Iter) Prev() (*Span, error) {
 func (i *Iter) Close() {}
 
 func (i *Iter) String() string {
-	return "fragmented-spans"
+	return "keyspan.Iter"
 }
 
 // WrapChildren implements FragmentIterator.
 func (i *Iter) WrapChildren(wrap WrapFn) {}
+
+// DebugTree is part of the FragmentIterator interface.
+func (i *Iter) DebugTree(tp treeprinter.Node) {
+	tp.Childf("%T(%p)", i, i)
+}
diff --git a/internal/keyspan/keyspanimpl/level_iter.go b/internal/keyspan/keyspanimpl/level_iter.go
index 9180bc88..e4b266e2 100644
--- a/internal/keyspan/keyspanimpl/level_iter.go
+++ b/internal/keyspan/keyspanimpl/level_iter.go
@@ -11,6 +11,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/manifest"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 )
 
 // TableNewSpanIter creates a new iterator for range key spans for the given
@@ -413,6 +414,14 @@ func (l *LevelIter) WrapChildren(wrap keyspan.WrapFn) {
 	l.wrapFn = wrap
 }
 
+// DebugTree is part of the FragmentIterator interface.
+func (l *LevelIter) DebugTree(tp treeprinter.Node) {
+	n := tp.Childf("%T(%p) %s", l, l, l.level)
+	if l.fileIter != nil {
+		l.fileIter.DebugTree(n)
+	}
+}
+
 func (l *LevelIter) setPosBeforeFile(f *manifest.FileMetadata) {
 	l.setPosInternal(f, beforeFile)
 }
diff --git a/internal/keyspan/keyspanimpl/merging_iter.go b/internal/keyspan/keyspanimpl/merging_iter.go
index 9ddf9b68..5cb07254 100644
--- a/internal/keyspan/keyspanimpl/merging_iter.go
+++ b/internal/keyspan/keyspanimpl/merging_iter.go
@@ -14,6 +14,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/manifest"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 )
 
 // TODO(jackson): Consider implementing an optimization to seek lower levels
@@ -1106,6 +1107,16 @@ func (m *MergingIter) WrapChildren(wrap keyspan.WrapFn) {
 	m.wrapFn = wrap
 }
 
+// DebugTree is part of the FragmentIterator interface.
+func (m *MergingIter) DebugTree(tp treeprinter.Node) {
+	n := tp.Childf("%T(%p)", m, m)
+	for i := range m.levels {
+		if iter := m.levels[i].iter; iter != nil {
+			m.levels[i].iter.DebugTree(n)
+		}
+	}
+}
+
 type mergingIterItem struct {
 	// boundKey points to the corresponding mergingIterLevel's `iterKey`.
 	*boundKey
diff --git a/internal/keyspan/logging_iter.go b/internal/keyspan/logging_iter.go
index 219fd825..a0a4b1ab 100644
--- a/internal/keyspan/logging_iter.go
+++ b/internal/keyspan/logging_iter.go
@@ -143,3 +143,11 @@ func (i *loggingIter) Close() {
 func (i *loggingIter) WrapChildren(wrap WrapFn) {
 	i.iter = wrap(i.iter)
 }
+
+// DebugTree is part of the FragmentIterator interface.
+func (i *loggingIter) DebugTree(tp treeprinter.Node) {
+	n := tp.Childf("%T(%p)", i, i)
+	if i.iter != nil {
+		i.iter.DebugTree(n)
+	}
+}
diff --git a/internal/keyspan/truncate.go b/internal/keyspan/truncate.go
index 898c51a8..20ea12da 100644
--- a/internal/keyspan/truncate.go
+++ b/internal/keyspan/truncate.go
@@ -7,6 +7,7 @@ package keyspan
 import (
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 )
 
 // Truncate creates a new iterator where every span in the supplied iterator is
@@ -171,3 +172,11 @@ func (i *truncatingIter) nextSpanWithinBounds(
 func (i *truncatingIter) WrapChildren(wrap WrapFn) {
 	i.iter = wrap(i.iter)
 }
+
+// DebugTree is part of the FragmentIterator interface.
+func (i *truncatingIter) DebugTree(tp treeprinter.Node) {
+	n := tp.Childf("%T(%p)", i, i)
+	if i.iter != nil {
+		i.iter.DebugTree(n)
+	}
+}
diff --git a/iterator.go b/iterator.go
index 564b0888..e0fd0959 100644
--- a/iterator.go
+++ b/iterator.go
@@ -21,6 +21,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/keyspan/keyspanimpl"
 	"github.com/cockroachdb/pebble/internal/manifest"
 	"github.com/cockroachdb/pebble/internal/rangekeystack"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 	"github.com/cockroachdb/pebble/sstable"
 	"github.com/cockroachdb/redact"
 )
@@ -3101,3 +3102,16 @@ func (i *Iterator) internalNext() (internalNextValidity, base.InternalKeyKind) {
 		panic("unreachable")
 	}
 }
+
+var _ base.IteratorDebug = (*Iterator)(nil)
+
+// DebugTree implements the base.IteratorDebug interface.
+func (i *Iterator) DebugTree(tp treeprinter.Node) {
+	n := tp.Childf("%T(%p)", i, i)
+	if i.iter != nil {
+		i.iter.DebugTree(n)
+	}
+	if i.pointIter != nil {
+		i.pointIter.DebugTree(n)
+	}
+}
diff --git a/level_iter.go b/level_iter.go
index a1dc823e..0e0390e2 100644
--- a/level_iter.go
+++ b/level_iter.go
@@ -14,6 +14,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/manifest"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 	"github.com/cockroachdb/pebble/sstable"
 )
 
@@ -945,6 +946,14 @@ func (l *levelIter) SetContext(ctx context.Context) {
 	}
 }
 
+// DebugTree is part of the InternalIterator interface.
+func (l *levelIter) DebugTree(tp treeprinter.Node) {
+	n := tp.Childf("%T(%p) %s", l, l, l.String())
+	if l.iter != nil {
+		l.iter.DebugTree(n)
+	}
+}
+
 func (l *levelIter) String() string {
 	if l.iterFile != nil {
 		return fmt.Sprintf("%s: fileNum=%s", l.level, l.iterFile.FileNum.String())
diff --git a/merging_iter.go b/merging_iter.go
index c67e655e..254d26b8 100644
--- a/merging_iter.go
+++ b/merging_iter.go
@@ -15,6 +15,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/keyspan"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 )
 
 type mergingIterLevel struct {
@@ -1326,6 +1327,16 @@ func (m *mergingIter) SetContext(ctx context.Context) {
 	}
 }
 
+// DebugTree is part of the InternalIterator interface.
+func (m *mergingIter) DebugTree(tp treeprinter.Node) {
+	n := tp.Childf("%T(%p)", m, m)
+	for i := range m.levels {
+		if iter := m.levels[i].iter; iter != nil {
+			iter.DebugTree(n)
+		}
+	}
+}
+
 func (m *mergingIter) DebugString() string {
 	var buf bytes.Buffer
 	sep := ""
diff --git a/range_keys.go b/range_keys.go
index 121a4894..8a91dbbb 100644
--- a/range_keys.go
+++ b/range_keys.go
@@ -12,6 +12,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/manifest"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 	"github.com/cockroachdb/pebble/sstable"
 )
 
@@ -715,6 +716,16 @@ func (i *lazyCombinedIter) SetContext(ctx context.Context) {
 	i.pointIter.SetContext(ctx)
 }
 
+// DebugTree is part of the InternalIterator interface.
+func (i *lazyCombinedIter) DebugTree(tp treeprinter.Node) {
+	n := tp.Childf("%T(%p)", i, i)
+	if i.combinedIterState.initialized {
+		i.parent.rangeKey.iiter.DebugTree(n)
+	} else {
+		i.pointIter.DebugTree(n)
+	}
+}
+
 func (i *lazyCombinedIter) String() string {
 	if i.combinedIterState.initialized {
 		return i.parent.rangeKey.iiter.String()
diff --git a/scan_internal.go b/scan_internal.go
index 76c89278..7f7ca327 100644
--- a/scan_internal.go
+++ b/scan_internal.go
@@ -15,6 +15,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/keyspan/keyspanimpl"
 	"github.com/cockroachdb/pebble/internal/manifest"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 	"github.com/cockroachdb/pebble/objstorage"
 	"github.com/cockroachdb/pebble/objstorage/remote"
 	"github.com/cockroachdb/pebble/sstable"
@@ -363,6 +364,12 @@ func (p *pointCollapsingIterator) SetContext(ctx context.Context) {
 	p.iter.SetContext(ctx)
 }
 
+// DebugTree is part of the InternalIterator interface.
+func (p *pointCollapsingIterator) DebugTree(tp treeprinter.Node) {
+	n := tp.Childf("%T(%p)", p, p)
+	p.iter.DebugTree(n)
+}
+
 // String implements the InternalIterator interface.
 func (p *pointCollapsingIterator) String() string {
 	return p.iter.String()
diff --git a/sstable/reader.go b/sstable/reader.go
index 7807ceec..98aba0f4 100644
--- a/sstable/reader.go
+++ b/sstable/reader.go
@@ -397,7 +397,7 @@ func (r *Reader) NewRawRangeDelIter(
 		return nil, err
 	}
 	transforms.ElideSameSeqNum = true
-	i, err := rowblk.NewFragmentIter(r.Compare, r.Split, h, transforms)
+	i, err := rowblk.NewFragmentIter(r.fileNum, r.Compare, r.Split, h, transforms)
 	if err != nil {
 		return nil, err
 	}
@@ -420,7 +420,7 @@ func (r *Reader) NewRawRangeKeyIter(
 	if err != nil {
 		return nil, err
 	}
-	i, err := rowblk.NewFragmentIter(r.Compare, r.Split, h, transforms)
+	i, err := rowblk.NewFragmentIter(r.fileNum, r.Compare, r.Split, h, transforms)
 	if err != nil {
 		return nil, err
 	}
diff --git a/sstable/reader_iter_single_lvl.go b/sstable/reader_iter_single_lvl.go
index b44c91f1..4d61b299 100644
--- a/sstable/reader_iter_single_lvl.go
+++ b/sstable/reader_iter_single_lvl.go
@@ -12,6 +12,7 @@ import (
 
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 	"github.com/cockroachdb/pebble/objstorage"
 	"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
 	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing"
@@ -1500,3 +1501,8 @@ func (i *singleLevelIterator) String() string {
 	}
 	return i.reader.fileNum.String()
 }
+
+// DebugTree is part of the InternalIterator interface.
+func (i *singleLevelIterator) DebugTree(tp treeprinter.Node) {
+	tp.Childf("%T(%p) fileNum=%s", i, i, i.String())
+}
diff --git a/sstable/reader_iter_two_lvl.go b/sstable/reader_iter_two_lvl.go
index 8beb9504..a9ac0786 100644
--- a/sstable/reader_iter_two_lvl.go
+++ b/sstable/reader_iter_two_lvl.go
@@ -11,6 +11,7 @@ import (
 
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 	"github.com/cockroachdb/pebble/objstorage"
 	"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
 	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing"
@@ -209,6 +210,11 @@ func (i *twoLevelIterator) String() string {
 	return i.reader.fileNum.String()
 }
 
+// DebugTree is part of the InternalIterator interface.
+func (i *twoLevelIterator) DebugTree(tp treeprinter.Node) {
+	tp.Childf("%T(%p) fileNum=%s", i, i, i.String())
+}
+
 // SeekGE implements internalIterator.SeekGE, as documented in the pebble
 // package. Note that SeekGE only checks the upper bound. It is up to the
 // caller to ensure that key is greater than or equal to the lower bound.
diff --git a/sstable/rowblk/rowblk_fragment_iter.go b/sstable/rowblk/rowblk_fragment_iter.go
index 3b211a49..b4648b5e 100644
--- a/sstable/rowblk/rowblk_fragment_iter.go
+++ b/sstable/rowblk/rowblk_fragment_iter.go
@@ -15,6 +15,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/rangedel"
 	"github.com/cockroachdb/pebble/internal/rangekey"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 	"github.com/cockroachdb/pebble/sstable/block"
 )
 
@@ -42,6 +43,9 @@ type fragmentIter struct {
 	span      keyspan.Span
 	dir       int8
 
+	// fileNum is used for logging/debugging.
+	fileNum base.DiskFileNum
+
 	// elideSameSeqnum, if true, returns only the first-occurring (in forward
 	// order) Key for each sequence number.
 	elideSameSeqnum bool
@@ -71,6 +75,7 @@ var fragmentBlockIterPool = sync.Pool{
 // NewFragmentIter returns a new keyspan iterator that iterates over a block's
 // spans.
 func NewFragmentIter(
+	fileNum base.DiskFileNum,
 	cmp base.Compare,
 	split base.Split,
 	blockHandle block.BufferHandle,
@@ -81,6 +86,7 @@ func NewFragmentIter(
 	// Use the i.keyBuf array to back the Keys slice to prevent an allocation
 	// when the spans contain few keys.
 	i.span.Keys = i.keyBuf[:0]
+	i.fileNum = fileNum
 	i.elideSameSeqnum = transforms.ElideSameSeqNum
 	i.syntheticPrefix = transforms.SyntheticPrefix
 	if transforms.SyntheticPrefix.IsSet() {
@@ -384,6 +390,11 @@ func (i *fragmentIter) String() string {
 // WrapChildren implements FragmentIterator.
 func (i *fragmentIter) WrapChildren(wrap keyspan.WrapFn) {}
 
+// DebugTree is part of the FragmentIterator interface.
+func (i *fragmentIter) DebugTree(tp treeprinter.Node) {
+	tp.Childf("%T(%p) fileNum=%s", i, i, i.fileNum)
+}
+
 func checkFragmentBlockIterator(obj interface{}) {
 	i := obj.(*fragmentIter)
 	if p := i.blockIter.Handle().Get(); p != nil {
diff --git a/sstable/rowblk/rowblk_iter.go b/sstable/rowblk/rowblk_iter.go
index c7da9340..e846db13 100644
--- a/sstable/rowblk/rowblk_iter.go
+++ b/sstable/rowblk/rowblk_iter.go
@@ -18,6 +18,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/manual"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 	"github.com/cockroachdb/pebble/sstable/block"
 )
 
@@ -1585,6 +1586,11 @@ func (i *Iter) Valid() bool {
 	return i.offset >= 0 && i.offset < i.restarts
 }
 
+// DebugTree is part of the InternalIterator interface.
+func (i *Iter) DebugTree(tp treeprinter.Node) {
+	tp.Childf("%T(%p)", i, i)
+}
+
 func (i *Iter) getRestart(idx int) int32 {
 	return int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*int32(idx):]))
 }
@@ -1879,6 +1885,11 @@ func (i *RawIter) Close() error {
 	return nil
 }
 
+// DebugTree is part of the InternalIterator interface.
+func (i *RawIter) DebugTree(tp treeprinter.Node) {
+	tp.Childf("%T(%p)", i, i)
+}
+
 func (i *RawIter) getRestart(idx int) int32 {
 	return int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*int32(idx):]))
 }


Test patch:
diff --git a/internal/base/test_utils.go b/internal/base/test_utils.go
index 0c87f903..b68b5817 100644
--- a/internal/base/test_utils.go
+++ b/internal/base/test_utils.go
@@ -10,6 +10,8 @@ import (
 	"io"
 	"strconv"
 	"strings"
+
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 )
 
 // NewDeletableSumValueMerger return a ValueMerger which computes the sum of its
@@ -251,6 +253,11 @@ func (f *FakeIter) SetBounds(lower, upper []byte) {
 // SetContext is part of the InternalIterator interface.
 func (f *FakeIter) SetContext(_ context.Context) {}
 
+// DebugTree is part of the InternalIterator interface.
+func (f *FakeIter) DebugTree(tp treeprinter.Node) {
+	tp.Childf("%T(%p)", f, f)
+}
+
 // ParseUserKeyBounds parses UserKeyBounds from a string representation of the
 // form "[foo, bar]" or "[foo, bar)".
 func ParseUserKeyBounds(s string) UserKeyBounds {
diff --git a/internal/itertest/probe.go b/internal/itertest/probe.go
index 0cfe836c..70622ac0 100644
--- a/internal/itertest/probe.go
+++ b/internal/itertest/probe.go
@@ -11,6 +11,7 @@ import (
 
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/dsl"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 )
 
 // OpKind indicates the type of iterator operation being performed.
@@ -248,6 +249,14 @@ func (p *probeIterator) SetContext(ctx context.Context) {
 	}
 }
 
+// DebugTree is part of the InternalIterator interface.
+func (p *probeIterator) DebugTree(tp treeprinter.Node) {
+	n := tp.Childf("%T(%p)", p, p)
+	if p.iter != nil {
+		p.iter.DebugTree(n)
+	}
+}
+
 func (p *probeIterator) String() string {
 	if p.iter != nil {
 		return fmt.Sprintf("probeIterator(%q)", p.iter.String())
diff --git a/internal/keyspan/interleaving_iter_test.go b/internal/keyspan/interleaving_iter_test.go
index f9d4df46..4d6ff575 100644
--- a/internal/keyspan/interleaving_iter_test.go
+++ b/internal/keyspan/interleaving_iter_test.go
@@ -16,6 +16,7 @@ import (
 	"github.com/cockroachdb/datadriven"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/testkeys"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 	"github.com/stretchr/testify/require"
 )
 
@@ -282,3 +283,8 @@ func (i *pointIterator) SetBounds(lower, upper []byte) {
 	i.lower, i.upper = lower, upper
 }
 func (i *pointIterator) SetContext(_ context.Context) {}
+
+// DebugTree is part of the InternalIterator interface.
+func (i *pointIterator) DebugTree(tp treeprinter.Node) {
+	tp.Childf("%T(%p)", i, i)
+}
diff --git a/internal/keyspan/test_utils.go b/internal/keyspan/test_utils.go
index 9a1040a4..bbff49a7 100644
--- a/internal/keyspan/test_utils.go
+++ b/internal/keyspan/test_utils.go
@@ -15,6 +15,7 @@ import (
 
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/dsl"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 )
 
 // This file contains testing facilities for Spans and FragmentIterators. It's
@@ -376,6 +377,14 @@ func (p *probeIterator) WrapChildren(wrap WrapFn) {
 	p.iter = wrap(p.iter)
 }
 
+// DebugTree is part of the FragmentIterator interface.
+func (p *probeIterator) DebugTree(tp treeprinter.Node) {
+	n := tp.Childf("%T(%p)", p, p)
+	if p.iter != nil {
+		p.iter.DebugTree(n)
+	}
+}
+
 // RunIterCmd evaluates a datadriven command controlling an internal
 // keyspan.FragmentIterator, writing the results of the iterator operations to
 // the provided writer.
@@ -560,3 +569,11 @@ func (i *invalidatingIter) Close() {
 func (i *invalidatingIter) WrapChildren(wrap WrapFn) {
 	i.iter = wrap(i.iter)
 }
+
+// DebugTree is part of the FragmentIterator interface.
+func (i *invalidatingIter) DebugTree(tp treeprinter.Node) {
+	n := tp.Childf("%T(%p)", i, i)
+	if i.iter != nil {
+		i.iter.DebugTree(n)
+	}
+}
diff --git a/keyspan_probe_test.go b/keyspan_probe_test.go
index dbe9fb5b..759ca3b2 100644
--- a/keyspan_probe_test.go
+++ b/keyspan_probe_test.go
@@ -15,6 +15,7 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/dsl"
 	"github.com/cockroachdb/pebble/internal/keyspan"
+	"github.com/cockroachdb/pebble/internal/treeprinter"
 )
 
 // This file contains testing facilities for Spans and FragmentIterators. It's
@@ -368,6 +369,14 @@ func (p *probeKeyspanIterator) WrapChildren(wrap keyspan.WrapFn) {
 	p.iter = wrap(p.iter)
 }
 
+// DebugTree is part of the FragmentIterator interface.
+func (p *probeKeyspanIterator) DebugTree(tp treeprinter.Node) {
+	n := tp.Childf("%T(%p)", p, p)
+	if p.iter != nil {
+		p.iter.DebugTree(n)
+	}
+}
+
 func (p *probeKeyspanIterator) Close() {
 	op := keyspanOp{Kind: opSpanClose}
 	if p.iter != nil {
diff --git a/sstable/rowblk/rowblk_fragment_iter_test.go b/sstable/rowblk/rowblk_fragment_iter_test.go
index a01ab442..8e484053 100644
--- a/sstable/rowblk/rowblk_fragment_iter_test.go
+++ b/sstable/rowblk/rowblk_fragment_iter_test.go
@@ -88,7 +88,7 @@ func TestBlockFragmentIterator(t *testing.T) {
 			transforms.SyntheticPrefix = []byte(syntheticPrefix)
 
 			blockHandle := block.CacheBufferHandle(c.Get(1, 0, 0))
-			i, err := NewFragmentIter(comparer.Compare, comparer.Split, blockHandle, transforms)
+			i, err := NewFragmentIter(0, comparer.Compare, comparer.Split, blockHandle, transforms)
 			defer i.Close()
 			require.NoError(t, err)