Repository: cockroachdb/pebble

Issue:
db: consider ingest value separation optimizations
When a sstable is ingested, all the table's values are in-place.

Value separation uses a 'MaxReferenceDepth' heuristic to determine when a compaction should write new blob files. If the ingested sstable is an input to a compaction alongside sstables with existing blob references, it's unlikely the comapction will decide to write new blob files and separate the ingested sstable's values to a blob file. It may take many compactions before the table's values are separated.

We could support ingesting sstables alongside blob files. Or we could adapt the heuristics to encourage these compactions with ingested sstables to write new blob files more eagerly.

Jira issue: PEBBLE-468

Epic CRDB-53453

Hints from issue comments:

Solution patch:
diff --git a/compaction.go b/compaction.go
index 7e30efb2..de7e37b5 100644
--- a/compaction.go
+++ b/compaction.go
@@ -250,7 +250,7 @@ type tableCompaction struct {
 	// b) rewrite blob files: The compaction will write eligible values to new
 	// blob files. This consumes more write bandwidth because all values are
 	// rewritten. However it restores locality.
-	getValueSeparation func(JobID, *tableCompaction) compact.ValueSeparation
+	getValueSeparation func(JobID, *tableCompaction, ValueStoragePolicy) compact.ValueSeparation
 
 	// startLevel is the level that is being compacted. Inputs from startLevel
 	// and outputLevel will be merged to produce a set of outputLevel files.
@@ -542,7 +542,7 @@ func (c *tableCompaction) makeInfo(jobID JobID) CompactionInfo {
 	return info
 }
 
-type getValueSeparation func(JobID, *tableCompaction) compact.ValueSeparation
+type getValueSeparation func(JobID, *tableCompaction, ValueStoragePolicy) compact.ValueSeparation
 
 // newCompaction constructs a compaction from the provided picked compaction.
 //
@@ -3300,10 +3300,7 @@ func (d *DB) runDefaultTableCompaction(
 	d.mu.Unlock()
 	defer d.mu.Lock()
 
-	// Determine whether we should separate values into blob files.
-	valueSeparation := c.getValueSeparation(jobID, c)
-
-	result := d.compactAndWrite(jobID, c, snapshots, valueSeparation)
+	result := d.compactAndWrite(jobID, c, snapshots)
 	if result.Err == nil {
 		ve, result.Err = c.makeVersionEdit(result)
 	}
@@ -3344,10 +3341,7 @@ func (d *DB) runDefaultTableCompaction(
 // compactAndWrite runs the data part of a compaction, where we set up a
 // compaction iterator and use it to write output tables.
 func (d *DB) compactAndWrite(
-	jobID JobID,
-	c *tableCompaction,
-	snapshots compact.Snapshots,
-	valueSeparation compact.ValueSeparation,
+	jobID JobID, c *tableCompaction, snapshots compact.Snapshots,
 ) (result compact.Result) {
 	suggestedCacheReaders := blob.SuggestedCachedReaders(len(c.inputs))
 	// Compactions use a pool of buffers to read blocks, avoiding polluting the
@@ -3442,25 +3436,36 @@ func (d *DB) compactAndWrite(
 	}
 	runner := compact.NewRunner(runnerCfg, iter)
 
-	var spanPolicyValid bool
-	var spanPolicy SpanPolicy
-	// If spanPolicyValid is true and spanPolicyEndKey is empty, then spanPolicy
-	// applies for the rest of the keyspace.
+	// Determine the value separation policy for this compaction.
+	// We pass the value storage span policy if one applies for the entire
+	// compaction keyspace in order to preserve blob references.
+	// Note that the value storage policy may change per table if
+	// there are different span policies in effect for this output
+	// range.
+	var compactionValueStoragePolicy ValueStoragePolicy
 	var spanPolicyEndKey []byte
+	var spanPolicy SpanPolicy
+	spanPolicy, spanPolicyEndKey, err = d.opts.Experimental.SpanPolicyFunc(c.bounds.Start)
+	if err != nil {
+		return runner.Finish().WithError(err)
+	}
+	if len(spanPolicyEndKey) == 0 || d.cmp(c.bounds.End.Key, spanPolicyEndKey) < 0 {
+		compactionValueStoragePolicy = spanPolicy.ValueStoragePolicy
+	}
 
+	valueSeparation := c.getValueSeparation(jobID, c, compactionValueStoragePolicy)
 	for runner.MoreDataToWrite() {
 		if c.cancel.Load() {
 			return runner.Finish().WithError(ErrCancelledCompaction)
 		}
 		// Create a new table.
 		firstKey := runner.FirstKey()
-		if !spanPolicyValid || (len(spanPolicyEndKey) > 0 && d.cmp(firstKey, spanPolicyEndKey) >= 0) {
+		if len(spanPolicyEndKey) > 0 && d.cmp(firstKey, spanPolicyEndKey) >= 0 {
 			var err error
 			spanPolicy, spanPolicyEndKey, err = d.opts.Experimental.SpanPolicyFunc(firstKey)
 			if err != nil {
 				return runner.Finish().WithError(err)
 			}
-			spanPolicyValid = true
 		}
 
 		writerOpts := d.makeWriterOptions(c.outputLevel.level)
diff --git a/db.go b/db.go
index 2f808573..7a27d63d 100644
--- a/db.go
+++ b/db.go
@@ -501,6 +501,12 @@ type DB struct {
 			// sstables.
 			cumulativePinnedCount uint64
 			cumulativePinnedSize  uint64
+
+			// The ongoing excises. These are under snapshots since this is only
+			// needed to coordinate with EventuallyFileOnlySnapshot creation.
+			ongoingExcises map[SeqNum]KeyRange
+			// Signalled when an ongoingExcise is removed.
+			ongoingExcisesRemovedCond *sync.Cond
 		}
 
 		tableStats struct {
@@ -2812,3 +2818,14 @@ func (d *DB) DebugCurrentVersion() *manifest.Version {
 	defer d.mu.Unlock()
 	return d.mu.versions.currentVersion()
 }
+
+func (d *DB) removeFromOngoingExcises(seqNum base.SeqNum) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	_, ok := d.mu.snapshots.ongoingExcises[seqNum]
+	if !ok {
+		panic(errors.AssertionFailedf("pebble: no ongoing excise for seqnum %d", seqNum))
+	}
+	delete(d.mu.snapshots.ongoingExcises, seqNum)
+	d.mu.snapshots.ongoingExcisesRemovedCond.Broadcast()
+}
diff --git a/ingest.go b/ingest.go
index 2ba54b20..1072ea76 100644
--- a/ingest.go
+++ b/ingest.go
@@ -1526,7 +1526,9 @@ func (d *DB) ingest(ctx context.Context, args ingestArgs) (IngestOperationStats,
 	// asFlushable indicates whether the sstable was ingested as a flushable.
 	var asFlushable bool
 	var waitFlushStart crtime.Mono
+	var assignedSeqNum SeqNum
 	prepare := func(seqNum base.SeqNum) {
+		assignedSeqNum = seqNum
 		// Note that d.commit.mu is held by commitPipeline when calling prepare.
 
 		// Determine the set of bounds we care about for the purpose of checking
@@ -1547,14 +1549,35 @@ func (d *DB) ingest(ctx context.Context, args ingestArgs) (IngestOperationStats,
 		}
 
 		d.mu.Lock()
-		defer d.mu.Unlock()
-
+		defer func() {
+			if args.ExciseSpan.Valid() && !asFlushable {
+				// This can conflict with a concurrent EFOS creation (unlike a
+				// flushable ingest, which does not destructively excise).
+				_, ok := d.mu.snapshots.ongoingExcises[seqNum]
+				if ok {
+					panic(errors.AssertionFailedf("pebble: excise with seqnum %s already in map", seqNum))
+				}
+				d.mu.snapshots.ongoingExcises[seqNum] = args.ExciseSpan
+			}
+			d.mu.Unlock()
+		}()
 		if args.ExciseSpan.Valid() {
 			// Check if any of the currently-open EventuallyFileOnlySnapshots
-			// overlap in key ranges with the excise span. If so, we need to
-			// check for memtable overlaps with all bounds of that
-			// EventuallyFileOnlySnapshot in addition to the ingestion's own
-			// bounds too.
+			// overlap in key ranges with the excise span. If so, we need to check
+			// for memtable overlaps with *all* bounds of that
+			// EventuallyFileOnlySnapshot in addition to the ingestion's own bounds
+			// too. This is needed because excise is a destructive operation that
+			// mutates the LSM in a non-additive way (unlike ingests and writes).
+			//
+			// Say there is an existing EFOS for the key-range [a, d) where a is in
+			// the memtable and visible to the EFOS, and a later excise wants to
+			// excise [c, g) and there is nothing overlapping with it in the
+			// memtable. By adding [a, d) to the overlap bounds, we ensure that the
+			// excise waits until all of that span is flushed from the memtables,
+			// which makes the EFOS transition to a FOS before the excise modifies
+			// the LSM. If the excise were to not wait, the [c, d) state that the
+			// EFOS needs in the LSM when transitioning to FOS will already have
+			// been excised.
 			overlapBounds = append(overlapBounds, exciseOverlapBounds(
 				d.cmp, &d.mu.snapshots.snapshotList, args.ExciseSpan, seqNum)...)
 		}
@@ -1735,6 +1758,13 @@ func (d *DB) ingest(ctx context.Context, args ingestArgs) (IngestOperationStats,
 	d.commit.ingestSem <- struct{}{}
 	d.commit.AllocateSeqNum(seqNumCount, prepare, apply)
 	<-d.commit.ingestSem
+	if args.ExciseSpan.Valid() && !asFlushable {
+		// NB: this must happen after the assignedSeqNum has become visible, so
+		// that any concurrent EFOS creation that acquires d.mu after the removal
+		// of this excise gets a visible seqnum after the excise. The
+		// assignedSeqNum becomes visible in AllocateSeqNum.
+		d.removeFromOngoingExcises(assignedSeqNum)
+	}
 
 	if err != nil {
 		if err2 := ingestCleanup(d.objProvider, loadResult.local); err2 != nil {
diff --git a/internal/manifest/table_metadata.go b/internal/manifest/table_metadata.go
index 6e0ef48f..9f02b19a 100644
--- a/internal/manifest/table_metadata.go
+++ b/internal/manifest/table_metadata.go
@@ -413,6 +413,13 @@ type TableBackingProperties struct {
 	TombstoneDenseBlocksRatio float64
 
 	CompressionStats block.CompressionStats
+
+	// ValueSeparationKind is the value separation policy used when writing the table.
+	ValueSeparationKind sstable.ValueSeparationKind
+	// ValueSeparationMinSize is the minimum value size for which values were
+	// separated when writing the table. This value is 0 if the policy used
+	// does not write blob files.
+	ValueSeparationMinSize uint64
 }
 
 // NumPointDeletions is the number of point deletions in the sstable. For virtual
@@ -447,6 +454,8 @@ func (b *TableBacking) PopulateProperties(props *sstable.Properties) *TableBacki
 		NumRangeKeyDels:            props.NumRangeKeyDels,
 		NumRangeKeySets:            props.NumRangeKeySets,
 		ValueBlocksSize:            props.ValueBlocksSize,
+		ValueSeparationKind:        sstable.ValueSeparationKind(props.ValueSeparationKind),
+		ValueSeparationMinSize:     props.ValueSeparationMinSize,
 	}
 	if props.NumDataBlocks != 0 {
 		b.props.TombstoneDenseBlocksRatio = float64(props.NumTombstoneDenseBlocks) / float64(props.NumDataBlocks)
diff --git a/open.go b/open.go
index dbd674c7..dd149389 100644
--- a/open.go
+++ b/open.go
@@ -13,6 +13,7 @@ import (
 	"math"
 	"os"
 	"slices"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -242,7 +243,9 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
 	d.mu.compact.cond.L = &d.mu.Mutex
 	d.mu.compact.inProgress = make(map[compaction]struct{})
 	d.mu.compact.noOngoingFlushStartTime = crtime.NowMono()
-	d.mu.snapshots.init()
+	d.mu.snapshots.snapshotList.init()
+	d.mu.snapshots.ongoingExcises = make(map[SeqNum]KeyRange)
+	d.mu.snapshots.ongoingExcisesRemovedCond = sync.NewCond(&d.mu.Mutex)
 	// logSeqNum is the next sequence number that will be assigned.
 	// Start assigning sequence numbers from base.SeqNumStart to leave
 	// room for reserved sequence numbers (see comments around
diff --git a/snapshot.go b/snapshot.go
index 91646c90..4dab187e 100644
--- a/snapshot.go
+++ b/snapshot.go
@@ -234,12 +234,83 @@ type EventuallyFileOnlySnapshot struct {
 }
 
 func (d *DB) makeEventuallyFileOnlySnapshot(keyRanges []KeyRange) *EventuallyFileOnlySnapshot {
-	isFileOnly := true
-
+	var snapshotSeqNum, exciseSeqNumToWaitFor base.SeqNum
+	// tryGetSnapshotSeqNum attempts to initialize a snapshotSeqNum. The
+	// snapshotSeqNum should be ignored if exciseSeqNumToWaitFor is > 0. In this
+	// case the caller should wait for this seqnum to become visible and call
+	// this function again.
+	tryGetSnapshotSeqNum := func() {
+		// In AllocateSeqNum, for an ingest-and-excise, some seqnums are
+		// allocated, say starting at N. Then AllocateSeqNum calls prepare, which
+		// acquires DB.mu and grabs all the existing EFOS that have not
+		// transitioned to FOS (which are in d.mu.snapshots.snapshotList) and
+		// overlap with the excise. After releasing DB.mu, in apply, the
+		// ingest-and-excise waits for all the previous EFOSs to transition to FOS
+		// (as a side effect of waiting for a memtable flush to complete). Note,
+		// the visible seqnum is still <= N-1, and will not be bumped to N until
+		// the ingest-and-excise completes. Any EFOS that gets created after
+		// prepare looked at the existing EFOSs, but before the ingest-and-excise
+		// completes, will be created with EventuallyFileOnlySnapshot.seqNum=N-1,
+		// and may not transition to FOS until after the excise, which is
+		// incorrect (the version at the transition to FOS has already experienced
+		// the excise). To avoid this incorrectness, tryGetSnapshotSeqNum is
+		// called in a loop, until there are no such ongoing excises. The loop can
+		// starve EFOS creation if the keyRanges keep overlapping with new ongoing
+		// ingest-and-excises. So EFOS should only be used when ingest-and-excises
+		// are rare over the keyRanges.
+		//
+		// Improving this starvation behavior would require this snapshot to
+		// register itself in a way that blocks future ingest-and-excises. The
+		// blocking would need to be done before allocating the seqnum, since
+		// blocking after can delay (latency sensitive) writes that get a seqnum
+		// later than the ingest-and-excise. Then the problem shifts to not
+		// starving the ingest-and-excise if EFOS creation is frequent. We observe
+		// that in the CockroachDB use case (a) ingest-and-excises are not very
+		// frequent, so EFOS starvation is unlikely, (b) EFOS creation is not
+		// latency sensitive. Hence, we ignore this starvation problem.
+		snapshotSeqNum = d.mu.versions.visibleSeqNum.Load()
+		// Check if any of the keyRanges overlap with an ongoing
+		// ingest-and-excise.
+		//
+		// NB: The zero seqnum cannot occur in practice, since base.SeqNumStart >
+		// 0.
+		exciseSeqNumToWaitFor = base.SeqNum(0)
+		for seqNum, span := range d.mu.snapshots.ongoingExcises {
+			if base.Visible(seqNum, snapshotSeqNum, base.SeqNumMax) {
+				// Skip this excise, since this is visible to the snapshot.
+				continue
+			}
+			// INVARIANT: seqNum >= snapshotSeqNum.
+			if seqNum <= exciseSeqNumToWaitFor {
+				// We are already waiting for a later excise.
+				continue
+			}
+			for i := range keyRanges {
+				if keyRanges[i].OverlapsKeyRange(d.cmp, span) {
+					exciseSeqNumToWaitFor = seqNum
+					break
+				}
+			}
+		}
+	}
 	d.mu.Lock()
 	defer d.mu.Unlock()
-	seqNum := d.mu.versions.visibleSeqNum.Load()
-	// Check if any of the keyRanges overlap with a memtable.
+	for {
+		// This call updates snapshotSeqNum and exciseSeqNumToWaitFor.
+		tryGetSnapshotSeqNum()
+		if exciseSeqNumToWaitFor == 0 {
+			break
+		}
+		for !base.Visible(exciseSeqNumToWaitFor, d.mu.versions.visibleSeqNum.Load(), base.SeqNumMax) {
+			d.mu.snapshots.ongoingExcisesRemovedCond.Wait()
+		}
+	}
+	isFileOnly := true
+	// Check if any of the keyRanges overlap with a memtable. It is possible
+	// (with very low probability) that all these memtable have seqnums later
+	// than seqNum, and the overlap is a false positive. This is harmless, and
+	// this EFOS will transition to FOS, when the false positive memtable is
+	// flushed.
 	for i := range d.mu.mem.queue {
 		d.mu.mem.queue[i].computePossibleOverlaps(func(bounded) shouldContinue {
 			isFileOnly = false
@@ -248,7 +319,7 @@ func (d *DB) makeEventuallyFileOnlySnapshot(keyRanges []KeyRange) *EventuallyFil
 	}
 	es := &EventuallyFileOnlySnapshot{
 		db:              d,
-		seqNum:          seqNum,
+		seqNum:          snapshotSeqNum,
 		protectedRanges: keyRanges,
 		closed:          make(chan struct{}),
 	}
@@ -258,7 +329,7 @@ func (d *DB) makeEventuallyFileOnlySnapshot(keyRanges []KeyRange) *EventuallyFil
 	} else {
 		s := &Snapshot{
 			db:     d,
-			seqNum: seqNum,
+			seqNum: snapshotSeqNum,
 		}
 		s.efos = es
 		es.mu.snap = s
diff --git a/sstable/properties.go b/sstable/properties.go
index d851f922..a0f3b2de 100644
--- a/sstable/properties.go
+++ b/sstable/properties.go
@@ -247,10 +247,6 @@ const (
 	// ValueSeparationSpanPolicy indicates that values were separated
 	// based on a span policy during writing.
 	ValueSeparationSpanPolicy
-
-	// ValueSeparationPreservedRefs indicates that existing blob references
-	// were preserved without creating new blob files.
-	ValueSeparationPreservedRefs
 )
 
 var (
diff --git a/value_separation.go b/value_separation.go
index 3a01b0ad..4ca2e01b 100644
--- a/value_separation.go
+++ b/value_separation.go
@@ -24,7 +24,7 @@ import (
 // ValueStorageLatencyTolerant.
 const latencyTolerantMinimumSize = 10
 
-var neverSeparateValues getValueSeparation = func(JobID, *tableCompaction) compact.ValueSeparation {
+var neverSeparateValues getValueSeparation = func(JobID, *tableCompaction, ValueStoragePolicy) compact.ValueSeparation {
 	return compact.NeverSeparateValues{}
 }
 
@@ -34,24 +34,32 @@ var neverSeparateValues getValueSeparation = func(JobID, *tableCompaction) compa
 //
 // It assumes that the compaction will write tables at d.TableFormat() or above.
 func (d *DB) determineCompactionValueSeparation(
-	jobID JobID, c *tableCompaction,
+	jobID JobID, c *tableCompaction, valueStorage ValueStoragePolicy,
 ) compact.ValueSeparation {
 	if d.FormatMajorVersion() < FormatValueSeparation ||
 		d.opts.Experimental.ValueSeparationPolicy == nil {
 		return compact.NeverSeparateValues{}
 	}
 	policy := d.opts.Experimental.ValueSeparationPolicy()
-	if !policy.Enabled {
+	if !policy.Enabled || valueStorage == ValueStorageLowReadLatency {
 		return compact.NeverSeparateValues{}
 	}
 
 	// We're allowed to write blob references. Determine whether we should carry
 	// forward existing blob references, or write new ones.
-	if writeBlobs, outputBlobReferenceDepth := shouldWriteBlobFiles(c, policy); !writeBlobs {
+	if writeBlobs, outputBlobReferenceDepth := shouldWriteBlobFiles(c, policy, valueStorage); !writeBlobs {
 		// This compaction should preserve existing blob references.
+		kind := sstable.ValueSeparationDefault
+		minSize := policy.MinimumSize
+		if valueStorage != ValueStorageDefault {
+			kind = sstable.ValueSeparationSpanPolicy
+			minSize = latencyTolerantMinimumSize
+		}
 		return &preserveBlobReferences{
-			inputBlobPhysicalFiles:   uniqueInputBlobMetadatas(&c.version.BlobFiles, c.inputs),
-			outputBlobReferenceDepth: outputBlobReferenceDepth,
+			inputBlobPhysicalFiles:     uniqueInputBlobMetadatas(&c.version.BlobFiles, c.inputs),
+			outputBlobReferenceDepth:   outputBlobReferenceDepth,
+			minimumValueSize:           minSize,
+			originalValueSeprationKind: kind,
 		}
 	}
 
@@ -84,7 +92,7 @@ func (d *DB) determineCompactionValueSeparation(
 // maximum blob reference depth to assign to output sstables (the actual value
 // may be lower iff the output table references fewer distinct blob files).
 func shouldWriteBlobFiles(
-	c *tableCompaction, policy ValueSeparationPolicy,
+	c *tableCompaction, policy ValueSeparationPolicy, valueStorage ValueStoragePolicy,
 ) (writeBlobs bool, referenceDepth manifest.BlobReferenceDepth) {
 	// Flushes will have no existing references to blob files and should write
 	// their values to new blob files.
@@ -113,6 +121,33 @@ func shouldWriteBlobFiles(
 	if inputReferenceDepth > manifest.BlobReferenceDepth(policy.MaxBlobReferenceDepth) {
 		return true, 0
 	}
+	// Compare policies used by each input file. If all input files have the
+	// same policy characteristics as the current one, then we can preserve
+	// existing blob references. This ensures that tables that were not written
+	// with value separation enabled will have their values written to new blob files.
+	for _, level := range c.inputs {
+		for t := range level.files.All() {
+			backingProps, backingPropsValid := t.TableBacking.Properties()
+			if !backingPropsValid {
+				continue
+			}
+			switch valueStorage {
+			case ValueStorageLowReadLatency:
+				// This case should be handled prior to calling this function,
+				// but include it here for completeness.
+				return false, inputReferenceDepth
+			case ValueStorageLatencyTolerant:
+				if backingProps.ValueSeparationMinSize != latencyTolerantMinimumSize {
+					return true, 0
+				}
+			default:
+				if int(backingProps.ValueSeparationMinSize) != policy.MinimumSize {
+					return true, 0
+				}
+			}
+		}
+	}
+
 	// Otherwise, we won't write any new blob files but will carry forward
 	// existing references.
 	return false, inputReferenceDepth
@@ -385,6 +420,13 @@ type preserveBlobReferences struct {
 	inputBlobPhysicalFiles   map[base.BlobFileID]*manifest.PhysicalBlobFile
 	outputBlobReferenceDepth manifest.BlobReferenceDepth
 
+	// minimumValueSize is the minimum size of values used by the value separation
+	// policy that was originally used to write the input sstables.
+	minimumValueSize int
+	// originalValueSeprationKind is the value separation policy that was originally used to
+	// write the input sstables.
+	originalValueSeprationKind sstable.ValueSeparationKind
+
 	// state
 	buf []byte
 	// currReferences holds the pending references that have been referenced by
@@ -412,11 +454,11 @@ func (vs *preserveBlobReferences) SetNextOutputConfig(config compact.ValueSepara
 
 // Kind implements the ValueSeparation interface.
 func (vs *preserveBlobReferences) Kind() sstable.ValueSeparationKind {
-	return sstable.ValueSeparationPreservedRefs
+	return vs.originalValueSeprationKind
 }
 
 // MinimumSize implements the ValueSeparation interface.
-func (vs *preserveBlobReferences) MinimumSize() int { return 0 }
+func (vs *preserveBlobReferences) MinimumSize() int { return vs.minimumValueSize }
 
 // EstimatedFileSize returns an estimate of the disk space consumed by the current
 // blob file if it were closed now.



Test patch:
diff --git a/data_test.go b/data_test.go
index 967adb10..cf46374c 100644
--- a/data_test.go
+++ b/data_test.go
@@ -912,7 +912,10 @@ func runDBDefineCmd(td *datadriven.TestData, opts *Options) (*DB, error) {
 	// all the tables, we can construct all the referenced blob files and add
 	// them to the final version edit.
 	valueSeparator := &defineDBValueSeparator{
-		pbr:   &preserveBlobReferences{},
+		pbr: &preserveBlobReferences{
+			originalValueSeprationKind: sstable.ValueSeparationDefault,
+			minimumValueSize:           d.opts.Experimental.ValueSeparationPolicy().MinimumSize,
+		},
 		metas: make(map[base.BlobFileID]*manifest.PhysicalBlobFile),
 	}
 
@@ -936,7 +939,7 @@ func runDBDefineCmd(td *datadriven.TestData, opts *Options) (*DB, error) {
 		if err != nil {
 			return err
 		}
-		c.getValueSeparation = func(JobID, *tableCompaction) compact.ValueSeparation {
+		c.getValueSeparation = func(JobID, *tableCompaction, ValueStoragePolicy) compact.ValueSeparation {
 			return valueSeparator
 		}
 		// NB: define allows the test to exactly specify which keys go
diff --git a/internal/manifest/table_metadata_test.go b/internal/manifest/table_metadata_test.go
index 8bef3c1a..aa13e93c 100644
--- a/internal/manifest/table_metadata_test.go
+++ b/internal/manifest/table_metadata_test.go
@@ -198,7 +198,7 @@ func TestTableMetadataSize(t *testing.T) {
 			structSize, tableMetadataSize)
 	}
 
-	const tableBackingSize = 160
+	const tableBackingSize = 176
 	if structSize := unsafe.Sizeof(TableBacking{}); structSize != tableBackingSize {
 		t.Errorf("TableBacking struct size (%d bytes) is not expected size (%d bytes)",
 			structSize, tableBackingSize)
diff --git a/snapshot_test.go b/snapshot_test.go
index 55816461..f3a9b4b3 100644
--- a/snapshot_test.go
+++ b/snapshot_test.go
@@ -384,11 +384,8 @@ func TestNewSnapshotRace(t *testing.T) {
 	require.NoError(t, d.Close())
 }
 
-// Test for https://github.com/cockroachdb/pebble/issues/5390.
+// Test for fix to https://github.com/cockroachdb/pebble/issues/5390.
 func TestEFOSAndExciseRace(t *testing.T) {
-	// Skipping since: panic: unexpected excise of an
-	// EventuallyFileOnlySnapshot's bounds.
-	t.Skip("#5390")
 	o := &Options{
 		FS:                          vfs.NewMem(),
 		L0CompactionThreshold:       10,
@@ -414,25 +411,33 @@ func TestEFOSAndExciseRace(t *testing.T) {
 	<-beforeIngestApply
 	// IngestAndExcise is blocked, after waiting for memtable flush.
 	//
-	// Create a EFOS that overlaps with the excise.
-	snap := d.NewEventuallyFileOnlySnapshot([]KeyRange{{Start: []byte("a"), End: []byte("d")}})
-	defer snap.Close()
-	checkIter := func() {
-		iter, err := snap.NewIter(nil)
-		// The iter sees a and c, since it precedes the excise.
-		require.NoError(t, err)
-		require.True(t, iter.First())
-		require.Equal(t, []byte("a"), iter.Key())
-		require.True(t, iter.Next())
-		require.Equal(t, []byte("c"), iter.Key())
-		require.False(t, iter.Next())
-		require.NoError(t, iter.Close())
-	}
-	checkIter()
-	// Unblock the excise.
+	// Create a EFOS that overlaps with the excise. It will block due to the
+	// ongoing excise.
+	snapDone := make(chan struct{})
+	go func() {
+		snap := d.NewEventuallyFileOnlySnapshot([]KeyRange{{Start: []byte("a"), End: []byte("d")}})
+		defer func() {
+			require.NoError(t, snap.Close())
+			snapDone <- struct{}{}
+		}()
+		checkIter := func() {
+			iter, err := snap.NewIter(nil)
+			// The iter only sees a, since it comes after the excise.
+			require.NoError(t, err)
+			require.True(t, iter.First())
+			require.Equal(t, []byte("a"), iter.Key())
+			require.False(t, iter.Next())
+			require.NoError(t, iter.Close())
+		}
+		checkIter()
+		// Flush and transition to FOS.
+		require.NoError(t, d.Flush())
+		require.NoError(t, snap.WaitForFileOnlySnapshot(context.Background(), 0))
+		checkIter()
+	}()
+	time.Sleep(100 * time.Millisecond)
+	// Unblock the excise, which eventually unblocks the snapshot creation.
 	waitForCh <- struct{}{}
-	// Flush and transition to FOS.
-	require.NoError(t, d.Flush())
-	require.NoError(t, snap.WaitForFileOnlySnapshot(context.Background(), 0))
-	checkIter()
+	// Wait for the snapshot checking to finish.
+	<-snapDone
 }
diff --git a/testdata/compaction/value_separation b/testdata/compaction/value_separation
index da42ee16..c889ea93 100644
--- a/testdata/compaction/value_separation
+++ b/testdata/compaction/value_separation
@@ -64,7 +64,7 @@ set c 6
 compact a-d
 ----
 L6:
-  000013:[a#0,SET-d#0,SET] seqnums:[0-0] points:[a#0,SET-d#0,SET] size:819 blobrefs:[(B000006: 1), (B000012: 2), (B000009: 1); depth:2]
+  000013:[a#0,SET-d#0,SET] seqnums:[0-0] points:[a#0,SET-d#0,SET] size:834 blobrefs:[(B000006: 1), (B000012: 2), (B000009: 1); depth:2]
 Blob files:
   B000006 physical:{000006 size:[92 (92B)] vals:[2 (2B)]}
   B000009 physical:{000009 size:[92 (92B)] vals:[2 (2B)]}
@@ -90,7 +90,7 @@ L6 blob-depth=3
   z.SET.0:blob{fileNum=100004 value=zoo}
 ----
 L6:
-  000004:[a#0,SET-z#0,SET] seqnums:[0-0] points:[a#0,SET-z#0,SET] size:821 blobrefs:[(B100002: 3), (B100003: 3), (B100004: 3); depth:3]
+  000004:[a#0,SET-z#0,SET] seqnums:[0-0] points:[a#0,SET-z#0,SET] size:836 blobrefs:[(B100002: 3), (B100003: 3), (B100004: 3); depth:3]
 Blob files:
   B100002 physical:{100002 size:[92 (92B)] vals:[3 (3B)]}
   B100003 physical:{100003 size:[92 (92B)] vals:[3 (3B)]}
@@ -108,7 +108,7 @@ flush
 L0.0:
   000006:[d#10,SET-e#11,SET] seqnums:[10-11] points:[d#10,SET-e#11,SET] size:795 blobrefs:[(B000007: 10); depth:1]
 L6:
-  000004:[a#0,SET-z#0,SET] seqnums:[0-0] points:[a#0,SET-z#0,SET] size:821 blobrefs:[(B100002: 3), (B100003: 3), (B100004: 3); depth:3]
+  000004:[a#0,SET-z#0,SET] seqnums:[0-0] points:[a#0,SET-z#0,SET] size:836 blobrefs:[(B100002: 3), (B100003: 3), (B100004: 3); depth:3]
 Blob files:
   B000007 physical:{000007 size:[100 (100B)] vals:[10 (10B)]}
   B100002 physical:{100002 size:[92 (92B)] vals:[3 (3B)]}
@@ -312,9 +312,9 @@ L0 blob-depth=3
   z.SET.1:blob{fileNum=100004 value=zoo}
 ----
 L0.1:
-  000004:[a#9,SET-d#9,SET] seqnums:[9-9] points:[a#9,SET-d#9,SET] size:790 blobrefs:[(B100001: 1); depth:1]
+  000004:[a#9,SET-d#9,SET] seqnums:[9-9] points:[a#9,SET-d#9,SET] size:805 blobrefs:[(B100001: 1); depth:1]
 L0.0:
-  000005:[a#1,SET-z#1,SET] seqnums:[1-1] points:[a#1,SET-z#1,SET] size:821 blobrefs:[(B100002: 3), (B100003: 3), (B100004: 3); depth:3]
+  000005:[a#1,SET-z#1,SET] seqnums:[1-1] points:[a#1,SET-z#1,SET] size:836 blobrefs:[(B100002: 3), (B100003: 3), (B100004: 3); depth:3]
 Blob files:
   B100001 physical:{100001 size:[90 (90B)] vals:[1 (1B)]}
   B100002 physical:{100002 size:[92 (92B)] vals:[3 (3B)]}
@@ -324,13 +324,15 @@ Blob files:
 compact a-z
 ----
 L1:
-  000006:[a#0,SET-z#0,SET] seqnums:[0-0] points:[a#0,SET-z#0,SET] size:840 blobrefs:[(B100002: 3), (B100001: 1), (B100003: 3), (B100004: 3); depth:4]
+  000006:[a#0,SET-z#0,SET] seqnums:[0-0] points:[a#0,SET-z#0,SET] size:855 blobrefs:[(B100002: 3), (B100001: 1), (B100003: 3), (B100004: 3); depth:4]
 Blob files:
   B100001 physical:{100001 size:[90 (90B)] vals:[1 (1B)]}
   B100002 physical:{100002 size:[92 (92B)] vals:[3 (3B)]}
   B100003 physical:{100003 size:[92 (92B)] vals:[3 (3B)]}
   B100004 physical:{100004 size:[92 (92B)] vals:[3 (3B)]}
 
+# Preserving blob references should mean that value separation
+# props are carried over to the new file.
 sstable-properties file=000006
 ----
 rocksdb.num.entries: 6
@@ -351,7 +353,8 @@ pebble.num.values.in.blob-files: 4
 rocksdb.property.collectors: [obsolete-key]
 rocksdb.compression: Snappy
 pebble.compression_stats: None:209
-pebble.value-separation.kind: 3
+pebble.value-separation.kind: 1
+pebble.value-separation.min-size: 1
 obsolete-key: hex:00
 
 # Construct an initial state with two non-overlapping files in L0, both with
@@ -371,8 +374,8 @@ L0 blob-depth=3
   z.SET.1:blob{fileNum=100004 value=zoo}
 ----
 L0.0:
-  000004:[a#9,SET-d#9,SET] seqnums:[9-9] points:[a#9,SET-d#9,SET] size:790 blobrefs:[(B100001: 1); depth:1]
-  000005:[e#1,SET-z#1,SET] seqnums:[1-1] points:[e#1,SET-z#1,SET] size:821 blobrefs:[(B100002: 3), (B100003: 3), (B100004: 3); depth:3]
+  000004:[a#9,SET-d#9,SET] seqnums:[9-9] points:[a#9,SET-d#9,SET] size:805 blobrefs:[(B100001: 1); depth:1]
+  000005:[e#1,SET-z#1,SET] seqnums:[1-1] points:[e#1,SET-z#1,SET] size:836 blobrefs:[(B100002: 3), (B100003: 3), (B100004: 3); depth:3]
 Blob files:
   B100001 physical:{100001 size:[90 (90B)] vals:[1 (1B)]}
   B100002 physical:{100002 size:[92 (92B)] vals:[3 (3B)]}
@@ -382,7 +385,7 @@ Blob files:
 compact a-z
 ----
 L1:
-  000006:[a#0,SET-z#0,SET] seqnums:[0-0] points:[a#0,SET-z#0,SET] size:844 blobrefs:[(B100001: 1), (B100002: 3), (B100003: 3), (B100004: 3); depth:3]
+  000006:[a#0,SET-z#0,SET] seqnums:[0-0] points:[a#0,SET-z#0,SET] size:867 blobrefs:[(B100001: 1), (B100002: 3), (B100003: 3), (B100004: 3); depth:3]
 Blob files:
   B100001 physical:{100001 size:[90 (90B)] vals:[1 (1B)]}
   B100002 physical:{100002 size:[92 (92B)] vals:[3 (3B)]}
@@ -437,7 +440,7 @@ Blob files:
 compact a-b
 ----
 L6:
-  000010:[a#0,SET-h#0,SET] seqnums:[0-0] points:[a#0,SET-h#0,SET] size:832 blobrefs:[(B000006: 53), (B000009: 6); depth:2]
+  000010:[a#0,SET-h#0,SET] seqnums:[0-0] points:[a#0,SET-h#0,SET] size:855 blobrefs:[(B000006: 53), (B000009: 6); depth:2]
 Blob files:
   B000006 physical:{000006 size:[156 (156B)] vals:[60 (60B)]}
   B000009 physical:{000009 size:[95 (95B)] vals:[6 (6B)]}
@@ -445,7 +448,7 @@ Blob files:
 auto-compact
 ----
 L6:
-  000010:[a#0,SET-h#0,SET] seqnums:[0-0] points:[a#0,SET-h#0,SET] size:832 blobrefs:[(B000006: 53), (B000009: 6); depth:2]
+  000010:[a#0,SET-h#0,SET] seqnums:[0-0] points:[a#0,SET-h#0,SET] size:855 blobrefs:[(B000006: 53), (B000009: 6); depth:2]
 Blob files:
   B000006 physical:{000011 size:[150 (150B)] vals:[53 (53B)]}
   B000009 physical:{000009 size:[95 (95B)] vals:[6 (6B)]}
@@ -457,15 +460,15 @@ LSM                             |    vtables   |   value sep   |        |   inge
 level       size | tables  size |  count  size |  refsz valblk |     in | tables  size |   r     w
 -----------------+--------------+--------------+---------------+--------+--------------+----------
    L0         0B |      0    0B |      0     0 |     0B     0B |   156B |      0    0B |   0 13.92
-   L6        1KB |      1  832B |      0     0 |   232B     0B |  1.6KB |      0    0B |   1  0.50
+   L6      1.1KB |      1  855B |      0     0 |   232B     0B |  1.6KB |      0    0B |   1  0.51
 -----------------+--------------+--------------+---------------+--------+--------------+----------
-total        1KB |      1  832B |      0     0 |   232B     0B |   156B |      0    0B |   1 20.25
+total      1.1KB |      1  855B |      0     0 |   232B     0B |   156B |      0    0B |   1 20.40
 
 COMPACTIONS               |     moved    |     multilevel    |     read     |       written
 level | score    ff   cff | tables  size |   top    in  read | tables  blob | tables  sstsz blobsz
 ------+-------------------+--------------+-------------------+--------------+---------------------
    L0 |     -     0     0 |      0    0B |    0B    0B    0B |     0B    0B |      2  1.6KB   502B
-   L6 |     -  0.00  0.00 |      0    0B |    0B    0B    0B |   310B    0B |      1   832B     0B
+   L6 |     -  0.00  0.00 |      0    0B |    0B    0B    0B |   310B    0B |      1   855B     0B
 ------+-------------------+--------------+-------------------+--------------+---------------------
 total |     -     -     - |      0    0B |    0B    0B    0B |   310B    0B |      3  2.6KB   502B
 
@@ -635,17 +638,17 @@ Blob files:
 compact a-zzzz
 ----
 L6:
-  000052:[a@1#0,SET-ckw@1#0,SET] seqnums:[0-0] points:[a@1#0,SET-ckw@1#0,SET] size:23100 blobrefs:[(B000007: 51136), (B000009: 51136), (B000011: 6592); depth:1]
-  000053:[ckx@1#0,SET-ewd@1#0,SET] seqnums:[0-0] points:[ckx@1#0,SET-ewd@1#0,SET] size:22436 blobrefs:[(B000011: 44544), (B000013: 51136), (B000015: 13824); depth:1]
-  000054:[ewe@1#0,SET-hgu@1#0,SET] seqnums:[0-0] points:[ewe@1#0,SET-hgu@1#0,SET] size:23223 blobrefs:[(B000015: 37312), (B000017: 51136), (B000019: 19968); depth:1]
-  000055:[hgv@1#0,SET-js@1#0,SET] seqnums:[0-0] points:[hgv@1#0,SET-js@1#0,SET] size:22562 blobrefs:[(B000019: 31168), (B000021: 51136), (B000023: 27072); depth:1]
-  000056:[jsa@1#0,SET-mcx@1#0,SET] seqnums:[0-0] points:[jsa@1#0,SET-mcx@1#0,SET] size:23094 blobrefs:[(B000023: 24128), (B000025: 51136), (B000027: 33600); depth:1]
-  000057:[mcy@1#0,SET-onw@1#0,SET] seqnums:[0-0] points:[mcy@1#0,SET-onw@1#0,SET] size:23041 blobrefs:[(B000027: 17600), (B000029: 51200), (B000031: 40128); depth:1]
-  000058:[onx@1#0,SET-qyv@1#0,SET] seqnums:[0-0] points:[onx@1#0,SET-qyv@1#0,SET] size:23000 blobrefs:[(B000031: 11008), (B000033: 51200), (B000035: 46720); depth:1]
-  000059:[qyw@1#0,SET-tjs@1#0,SET] seqnums:[0-0] points:[qyw@1#0,SET-tjs@1#0,SET] size:23112 blobrefs:[(B000035: 4416), (B000037: 51200), (B000039: 51136), (B000041: 2112); depth:1]
-  000060:[tjt@1#0,SET-vuj@1#0,SET] seqnums:[0-0] points:[tjt@1#0,SET-vuj@1#0,SET] size:23520 blobrefs:[(B000041: 49088), (B000043: 51200), (B000045: 8128); depth:1]
-  000061:[vuk@1#0,SET-yez@1#0,SET] seqnums:[0-0] points:[vuk@1#0,SET-yez@1#0,SET] size:23537 blobrefs:[(B000045: 43008), (B000047: 51200), (B000049: 14144); depth:1]
-  000062:[yf@1#0,SET-zzz@1#0,SET] seqnums:[0-0] points:[yf@1#0,SET-zzz@1#0,SET] size:17780 blobrefs:[(B000049: 36992), (B000051: 44288); depth:1]
+  000052:[a@1#0,SET-ckw@1#0,SET] seqnums:[0-0] points:[a@1#0,SET-ckw@1#0,SET] size:23115 blobrefs:[(B000007: 51136), (B000009: 51136), (B000011: 6592); depth:1]
+  000053:[ckx@1#0,SET-ewd@1#0,SET] seqnums:[0-0] points:[ckx@1#0,SET-ewd@1#0,SET] size:22451 blobrefs:[(B000011: 44544), (B000013: 51136), (B000015: 13824); depth:1]
+  000054:[ewe@1#0,SET-hgu@1#0,SET] seqnums:[0-0] points:[ewe@1#0,SET-hgu@1#0,SET] size:23238 blobrefs:[(B000015: 37312), (B000017: 51136), (B000019: 19968); depth:1]
+  000055:[hgv@1#0,SET-js@1#0,SET] seqnums:[0-0] points:[hgv@1#0,SET-js@1#0,SET] size:22577 blobrefs:[(B000019: 31168), (B000021: 51136), (B000023: 27072); depth:1]
+  000056:[jsa@1#0,SET-mcx@1#0,SET] seqnums:[0-0] points:[jsa@1#0,SET-mcx@1#0,SET] size:23109 blobrefs:[(B000023: 24128), (B000025: 51136), (B000027: 33600); depth:1]
+  000057:[mcy@1#0,SET-onw@1#0,SET] seqnums:[0-0] points:[mcy@1#0,SET-onw@1#0,SET] size:23056 blobrefs:[(B000027: 17600), (B000029: 51200), (B000031: 40128); depth:1]
+  000058:[onx@1#0,SET-qyv@1#0,SET] seqnums:[0-0] points:[onx@1#0,SET-qyv@1#0,SET] size:23015 blobrefs:[(B000031: 11008), (B000033: 51200), (B000035: 46720); depth:1]
+  000059:[qyw@1#0,SET-tjs@1#0,SET] seqnums:[0-0] points:[qyw@1#0,SET-tjs@1#0,SET] size:23127 blobrefs:[(B000035: 4416), (B000037: 51200), (B000039: 51136), (B000041: 2112); depth:1]
+  000060:[tjt@1#0,SET-vuj@1#0,SET] seqnums:[0-0] points:[tjt@1#0,SET-vuj@1#0,SET] size:23535 blobrefs:[(B000041: 49088), (B000043: 51200), (B000045: 8128); depth:1]
+  000061:[vuk@1#0,SET-yez@1#0,SET] seqnums:[0-0] points:[vuk@1#0,SET-yez@1#0,SET] size:23552 blobrefs:[(B000045: 43008), (B000047: 51200), (B000049: 14144); depth:1]
+  000062:[yf@1#0,SET-zzz@1#0,SET] seqnums:[0-0] points:[yf@1#0,SET-zzz@1#0,SET] size:17795 blobrefs:[(B000049: 36992), (B000051: 44288); depth:1]
 Blob files:
   B000007 physical:{000007 size:[53138 (52KB)] vals:[51136 (50KB)]}
   B000009 physical:{000009 size:[53138 (52KB)] vals:[51136 (50KB)]}
@@ -676,8 +679,8 @@ excise-dryrun b c
 ----
 would excise 1 files.
   del-table:     L6 000052
-  add-table:     L6 000063(000052):[a@1#0,SET-azz@1#0,SET] seqnums:[0-0] points:[a@1#0,SET-azz@1#0,SET] size:11748(23100) blobrefs:[(B000007: 26006), (B000009: 26006), (B000011: 3352); depth:1]
-  add-table:     L6 000064(000052):[c@1#0,SET-ckw@1#0,SET] seqnums:[0-0] points:[c@1#0,SET-ckw@1#0,SET] size:6302(23100) blobrefs:[(B000007: 13950), (B000009: 13950), (B000011: 1798); depth:1]
+  add-table:     L6 000063(000052):[a@1#0,SET-azz@1#0,SET] seqnums:[0-0] points:[a@1#0,SET-azz@1#0,SET] size:11748(23115) blobrefs:[(B000007: 25989), (B000009: 25989), (B000011: 3350); depth:1]
+  add-table:     L6 000064(000052):[c@1#0,SET-ckw@1#0,SET] seqnums:[0-0] points:[c@1#0,SET-ckw@1#0,SET] size:6302(23115) blobrefs:[(B000007: 13941), (B000009: 13941), (B000011: 1797); depth:1]
   add-backing:   000052
 
 
@@ -817,3 +820,112 @@ pebble.compression_stats: None:56,Snappy:160/194
 pebble.value-separation.kind: 2
 pebble.value-separation.min-size: 10
 obsolete-key: hex:00
+
+batch
+set f helloworld
+set g helloworld!
+----
+
+flush
+----
+L0.1:
+  000010:[f#36,SET-g#37,SET] seqnums:[36-37] points:[f#36,SET-g#37,SET] size:796 blobrefs:[(B000011: 21); depth:1]
+L0.0:
+  000005:[a#10,SET-e#14,SET] seqnums:[10-14] points:[a#10,SET-e#14,SET] size:773
+  000006:[f#15,SET-n#23,SET] seqnums:[15-23] points:[f#15,SET-n#23,SET] size:880 blobrefs:[(B000007: 63); depth:1]
+  000008:[o#24,SET-z#35,SET] seqnums:[24-35] points:[o#24,SET-z#35,SET] size:818
+Blob files:
+  B000007 physical:{000007 size:[115 (115B)] vals:[63 (63B)]}
+  B000011 physical:{000011 size:[111 (111B)] vals:[21 (21B)]}
+
+
+# Compacting key range with the same value separation span policy should allow preserving blob references.
+compact f-n
+----
+L0.0:
+  000005:[a#10,SET-e#14,SET] seqnums:[10-14] points:[a#10,SET-e#14,SET] size:773
+  000008:[o#24,SET-z#35,SET] seqnums:[24-35] points:[o#24,SET-z#35,SET] size:818
+L6:
+  000012:[f#0,SET-n#0,SET] seqnums:[0-0] points:[f#0,SET-n#0,SET] size:872 blobrefs:[(B000011: 21), (B000007: 52); depth:2]
+Blob files:
+  B000007 physical:{000007 size:[115 (115B)] vals:[63 (63B)]}
+  B000011 physical:{000011 size:[111 (111B)] vals:[21 (21B)]}
+
+# Compacting key range with a mix of value separation span policies should rewrite blob references.
+# Note that this should not occur in practice because L0 compactions are built around table bounds.
+batch
+set f frolick
+set g garden
+set h holiday
+----
+
+flush
+----
+L0.0:
+  000005:[a#10,SET-e#14,SET] seqnums:[10-14] points:[a#10,SET-e#14,SET] size:773
+  000014:[f#38,SET-h#40,SET] seqnums:[38-40] points:[f#38,SET-h#40,SET] size:735
+  000008:[o#24,SET-z#35,SET] seqnums:[24-35] points:[o#24,SET-z#35,SET] size:818
+L6:
+  000012:[f#0,SET-n#0,SET] seqnums:[0-0] points:[f#0,SET-n#0,SET] size:872 blobrefs:[(B000011: 21), (B000007: 52); depth:2]
+Blob files:
+  B000007 physical:{000007 size:[115 (115B)] vals:[63 (63B)]}
+  B000011 physical:{000011 size:[111 (111B)] vals:[21 (21B)]}
+
+compact f-z
+----
+L0.0:
+  000005:[a#10,SET-e#14,SET] seqnums:[10-14] points:[a#10,SET-e#14,SET] size:773
+L6:
+  000015:[f#0,SET-n#0,SET] seqnums:[0-0] points:[f#0,SET-n#0,SET] size:871 blobrefs:[(B000016: 42); depth:1]
+  000017:[o#0,SET-z#0,SET] seqnums:[0-0] points:[o#0,SET-z#0,SET] size:792
+Blob files:
+  B000016 physical:{000016 size:[121 (121B)] vals:[42 (42B)]}
+
+
+sstable-properties file=000015
+----
+rocksdb.num.entries: 9
+rocksdb.raw.key.size: 81
+rocksdb.raw.value.size: 72
+rocksdb.deleted.keys: 0
+rocksdb.num.range-deletions: 0
+rocksdb.num.data.blocks: 1
+rocksdb.comparator: leveldb.BytewiseComparator
+rocksdb.data.size: 156
+rocksdb.filter.size: 0
+rocksdb.index.size: 36
+rocksdb.block.based.table.index.type: 0
+pebble.colblk.schema: DefaultKeySchema(leveldb.BytewiseComparator,16)
+rocksdb.merge.operator: pebble.concatenate
+rocksdb.merge.operands: 0
+pebble.num.values.in.blob-files: 4
+rocksdb.property.collectors: [obsolete-key]
+rocksdb.compression: Snappy
+pebble.compression_stats: None:56,Snappy:151/178
+pebble.value-separation.kind: 2
+pebble.value-separation.min-size: 10
+obsolete-key: hex:00
+
+
+sstable-properties file=000017
+----
+rocksdb.num.entries: 12
+rocksdb.raw.key.size: 108
+rocksdb.raw.value.size: 104
+rocksdb.deleted.keys: 0
+rocksdb.num.range-deletions: 0
+rocksdb.num.data.blocks: 1
+rocksdb.comparator: leveldb.BytewiseComparator
+rocksdb.data.size: 137
+rocksdb.filter.size: 0
+rocksdb.index.size: 36
+rocksdb.block.based.table.index.type: 0
+pebble.colblk.schema: DefaultKeySchema(leveldb.BytewiseComparator,16)
+rocksdb.merge.operator: pebble.concatenate
+rocksdb.merge.operands: 0
+rocksdb.property.collectors: [obsolete-key]
+rocksdb.compression: Snappy
+pebble.compression_stats: None:36,Snappy:132/225
+pebble.value-separation.kind: 1
+pebble.value-separation.min-size: 1024
+obsolete-key: hex:00
diff --git a/testdata/compaction_picker_scores b/testdata/compaction_picker_scores
index 45160f2a..77864a2f 100644
--- a/testdata/compaction_picker_scores
+++ b/testdata/compaction_picker_scores
@@ -331,13 +331,13 @@ Blob files:
 scores wait-for-compaction=completion
 ----
 Level    Size     Score     Fill factor    Compensated fill factor
-L0       1.0KB    151.53    2.00           2.00
+L0       1.0KB    149.45    2.00           2.00
 L1       0B       0.00      0.00           0.00
 L2       0B       0.00      0.00           0.00
 L3       0B       0.00      0.00           0.00
 L4       0B       0.00      0.00           0.00
 L5       0B       0.00      0.00           0.00
-L6       865B     0.00      0.01           0.01
+L6       877B     0.00      0.01           0.01
 
 # This attempt to compact should chose to rewrite the blob file B000006 *AND*
 # compact out of L0.
diff --git a/testdata/metrics b/testdata/metrics
index 6ca8c409..df1723b6 100644
--- a/testdata/metrics
+++ b/testdata/metrics
@@ -702,15 +702,15 @@ LSM                             |    vtables   |   value sep   |        |   inge
 level       size | tables  size |  count  size |  refsz valblk |     in | tables  size |   r     w
 -----------------+--------------+--------------+---------------+--------+--------------+----------
    L0         0B |      0    0B |      0     0 |     0B     0B |   165B |      0    0B |   0 57.40
-   L6      7.9KB |      9 6.7KB |      0     0 |  1.2KB     0B |  6.8KB |      0    0B |   1  0.98
+   L6        8KB |      9 6.8KB |      0     0 |  1.2KB     0B |  6.8KB |      0    0B |   1  1.00
 -----------------+--------------+--------------+---------------+--------+--------------+----------
-total      7.9KB |      9 6.7KB |      0     0 |  1.2KB     0B |   165B |      0    0B |   1 100.1
+total        8KB |      9 6.8KB |      0     0 |  1.2KB     0B |   165B |      0    0B |   1 100.6
 
 COMPACTIONS               |     moved    |     multilevel    |     read     |       written
 level | score    ff   cff | tables  size |   top    in  read | tables  blob | tables  sstsz blobsz
 ------+-------------------+--------------+-------------------+--------------+---------------------
    L0 |     -     0     0 |      0    0B |    0B    0B    0B |     0B    0B |      9  6.8KB  2.4KB
-   L6 |     -  0.00  0.00 |      0    0B |    0B    0B    0B |  5.4KB    0B |      9  6.7KB     0B
+   L6 |     -  0.00  0.00 |      0    0B |    0B    0B    0B |  5.4KB    0B |      9  6.8KB     0B
 ------+-------------------+--------------+-------------------+--------------+---------------------
 total |     -     -     - |      0    0B |    0B    0B    0B |  5.4KB    0B |     18   14KB  2.4KB
 
@@ -863,15 +863,15 @@ LSM                             |    vtables   |   value sep   |        |   inge
 level       size | tables  size |  count  size |  refsz valblk |     in | tables  size |   r     w
 -----------------+--------------+--------------+---------------+--------+--------------+----------
    L0        4KB |      6   4KB |      0     0 |     0B     0B |   211B |      3 1.9KB |   2 54.94
-   L6      7.9KB |      9 6.7KB |      0     0 |  1.2KB     0B |  6.8KB |      0    0B |   1  0.98
+   L6        8KB |      9 6.8KB |      0     0 |  1.2KB     0B |  6.8KB |      0    0B |   1  1.00
 -----------------+--------------+--------------+---------------+--------+--------------+----------
-total       12KB |     15  11KB |      0     0 |  1.2KB     0B |  2.1KB |      3 1.9KB |   3  9.49
+total       12KB |     15  11KB |      0     0 |  1.2KB     0B |  2.1KB |      3 1.9KB |   3  9.53
 
 COMPACTIONS               |     moved    |     multilevel    |     read     |       written
 level | score    ff   cff | tables  size |   top    in  read | tables  blob | tables  sstsz blobsz
 ------+-------------------+--------------+-------------------+--------------+---------------------
    L0 |     -  0.50  0.50 |      0    0B |    0B    0B    0B |     0B    0B |     12  8.9KB  2.4KB
-   L6 |     -  0.00  0.00 |      0    0B |    0B    0B    0B |  5.4KB    0B |      9  6.7KB     0B
+   L6 |     -  0.00  0.00 |      0    0B |    0B    0B    0B |  5.4KB    0B |      9  6.8KB     0B
 ------+-------------------+--------------+-------------------+--------------+---------------------
 total |     -     -     - |      0    0B |    0B    0B    0B |  5.4KB    0B |     21   18KB  2.4KB
 
@@ -986,15 +986,15 @@ LSM                             |    vtables   |   value sep   |        |   inge
 level       size | tables  size |  count  size |  refsz valblk |     in | tables  size |   r     w
 -----------------+--------------+--------------+---------------+--------+--------------+----------
    L0      8.8KB |     13 8.8KB |      0     0 |     0B     0B |   277B |      3 1.9KB |   2 59.71
-   L6      7.9KB |      9 6.7KB |      0     0 |  1.2KB     0B |  6.8KB |      0    0B |   1  0.98
+   L6        8KB |      9 6.8KB |      0     0 |  1.2KB     0B |  6.8KB |      0    0B |   1  1.00
 -----------------+--------------+--------------+---------------+--------+--------------+----------
-total       17KB |     22  16KB |      0     0 |  1.2KB     0B |  2.2KB |      3 1.9KB |   3 11.45
+total       17KB |     22  16KB |      0     0 |  1.2KB     0B |  2.2KB |      3 1.9KB |   3 11.49
 
 COMPACTIONS               |     moved    |     multilevel    |     read     |       written
 level | score    ff   cff | tables  size |   top    in  read | tables  blob | tables  sstsz blobsz
 ------+-------------------+--------------+-------------------+--------------+---------------------
    L0 |     -  0.50  0.50 |      0    0B |    0B    0B    0B |     0B    0B |     19   14KB  2.4KB
-   L6 |     -  0.00  0.00 |      0    0B |    0B    0B    0B |  5.4KB    0B |      9  6.7KB     0B
+   L6 |     -  0.00  0.00 |      0    0B |    0B    0B    0B |  5.4KB    0B |      9  6.8KB     0B
 ------+-------------------+--------------+-------------------+--------------+---------------------
 total |     -     -     - |      0    0B |    0B    0B    0B |  5.4KB    0B |     28   23KB  2.4KB
 
@@ -1121,15 +1121,15 @@ LSM                             |    vtables   |   value sep   |        |   inge
 level       size | tables  size |  count  size |  refsz valblk |     in | tables  size |   r     w
 -----------------+--------------+--------------+---------------+--------+--------------+----------
    L0      7.4KB |     11 7.4KB |      0     0 |     0B     0B |   277B |      3 1.9KB |   2 59.71
-   L6      8.6KB |     10 7.4KB |      0     0 |  1.2KB     0B |  6.8KB |      1  655B |   1  0.98
+   L6      8.7KB |     10 7.4KB |      0     0 |  1.2KB     0B |  6.8KB |      1  655B |   1  1.00
 -----------------+--------------+--------------+---------------+--------+--------------+----------
-total       16KB |     21  15KB |      0     0 |  1.2KB     0B |  2.8KB |      4 2.6KB |   3  9.09
+total       16KB |     21  15KB |      0     0 |  1.2KB     0B |  2.8KB |      4 2.6KB |   3  9.11
 
 COMPACTIONS               |     moved    |     multilevel    |     read     |       written
 level | score    ff   cff | tables  size |   top    in  read | tables  blob | tables  sstsz blobsz
 ------+-------------------+--------------+-------------------+--------------+---------------------
    L0 |     -  0.50  0.50 |      0    0B |    0B    0B    0B |     0B    0B |     19   14KB  2.4KB
-   L6 |     -  0.00  0.00 |      0    0B |    0B    0B    0B |  5.4KB    0B |      9  6.7KB     0B
+   L6 |     -  0.00  0.00 |      0    0B |    0B    0B    0B |  5.4KB    0B |      9  6.8KB     0B
 ------+-------------------+--------------+-------------------+--------------+---------------------
 total |     -     -     - |      0    0B |    0B    0B    0B |  5.4KB    0B |     28   23KB  2.4KB
 
@@ -1299,7 +1299,7 @@ level       size | tables  size |  count  size |  refsz valblk |     in | tables
    L0         0B |      0    0B |      0     0 |     0B     0B |   277B |      3 1.9KB |   0 59.71
    L6       14KB |     18  13KB |      0     0 |  1.2KB     0B |   14KB |      2 1.3KB |   1  0.85
 -----------------+--------------+--------------+---------------+--------+--------------+----------
-total       14KB |     18  13KB |      0     0 |  1.2KB     0B |  3.5KB |      5 3.2KB |   1  8.99
+total       14KB |     18  13KB |      0     0 |  1.2KB     0B |  3.5KB |      5 3.2KB |   1  9.01
 
 COMPACTIONS               |     moved    |     multilevel    |     read     |       written
 level | score    ff   cff | tables  size |   top    in  read | tables  blob | tables  sstsz blobsz
diff --git a/value_separation_test.go b/value_separation_test.go
index f4953da1..e6bf4d98 100644
--- a/value_separation_test.go
+++ b/value_separation_test.go
@@ -220,7 +220,7 @@ func (vs *defineDBValueSeparator) Kind() sstable.ValueSeparationKind {
 }
 
 // MinimumSize implements the ValueSeparation interface.
-func (vs *defineDBValueSeparator) MinimumSize() int { return 0 }
+func (vs *defineDBValueSeparator) MinimumSize() int { return vs.pbr.MinimumSize() }
 
 // EstimatedFileSize returns an estimate of the disk space consumed by the current
 // blob file if it were closed now.