Repository: cockroachdb/pebble

Issue:
make tombstone compaction threshold configurable
On some workloads, tombstone compactions can lead to increased resource usage with no visible benefit on latencies. We need the threshold to be configurable so we can easily turn it down or off in these cases.

Jira issue: PEBBLE-1225

Discussion:
@ever0de: If I'm understanding this correctly, I'd like to try implementing it. Could you let me know which threshold you're referring to?

1.  `TombstoneDenseCompactionThreshold` (default: 0.10) - [Already configurable](https://github.com/cockroachdb/pebble/blob/fcb941e0785b472a77a4076b32ee958e8c89cc14/options.go#L627-L637) since(https://github.com/cockroachdb/pebble/commit/28840262)

2.  `maxOverlappingRatio` (default: 40.0) - [Currently hardcoded](https://github.com/cockroachdb/pebble/blob/3dd8e6f3304f35209142eade45dde1da99a3bdad/compaction_picker.go#L1838-L1844) since ([fcb941e0](https://github.com/cockroachdb/pebble/commit/fcb941e0))

I suspect you mean 2 based on your description.@RaduBerinde: I was thinking of TombstoneDenseCompactionThreshold. I should have said *dynamically* reconfigurable (making it a function).

Solution patch:
diff --git a/compaction.go b/compaction.go
index e346bfd2..3f8990df 100644
--- a/compaction.go
+++ b/compaction.go
@@ -3448,10 +3448,6 @@ func (c *tableCompaction) makeVersionEdit(result compact.Result) (*manifest.Vers
 	// TODO(jackson):  This BytesRead value does not include any blob files
 	// written. It either should, or we should add a separate metric.
 	outputMetrics.TableBytesRead = c.outputLevel.files.TableSizeSum()
-	outputMetrics.BlobBytesCompacted = result.Stats.CumulativeBlobFileSize
-	if c.flush.flushables != nil {
-		outputMetrics.BlobBytesFlushed = result.Stats.CumulativeBlobFileSize
-	}
 	if len(c.extraLevels) > 0 {
 		outputMetrics.TableBytesIn += c.extraLevels[0].files.TableSizeSum()
 	}
@@ -3459,6 +3455,9 @@ func (c *tableCompaction) makeVersionEdit(result compact.Result) (*manifest.Vers
 
 	if len(c.flush.flushables) == 0 {
 		c.metrics.perLevel.level(c.startLevel.level)
+		outputMetrics.BlobBytesCompacted = result.Stats.CumulativeBlobFileSize
+	} else {
+		outputMetrics.BlobBytesFlushed = result.Stats.CumulativeBlobFileSize
 	}
 	if len(c.extraLevels) > 0 {
 		c.metrics.perLevel.level(c.extraLevels[0].level)
diff --git a/compaction_picker.go b/compaction_picker.go
index 8151b9f3..6287e150 100644
--- a/compaction_picker.go
+++ b/compaction_picker.go
@@ -1722,7 +1722,8 @@ func (p *compactionPickerByScore) pickBlobFileRewriteCompaction(
 func (p *compactionPickerByScore) pickTombstoneDensityCompaction(
 	env compactionEnv,
 ) (pc *pickedTableCompaction) {
-	if p.opts.Experimental.TombstoneDenseCompactionThreshold <= 0 {
+	threshold := p.opts.Experimental.TombstoneDenseCompactionThreshold()
+	if threshold <= 0 {
 		// Tombstone density compactions are disabled.
 		return nil
 	}
@@ -1745,7 +1746,7 @@ func (p *compactionPickerByScore) pickTombstoneDensityCompaction(
 			if f.IsCompacting() || !f.StatsValid() || f.Size == 0 {
 				continue
 			}
-			if f.Stats.TombstoneDenseBlocksRatio < p.opts.Experimental.TombstoneDenseCompactionThreshold {
+			if f.Stats.TombstoneDenseBlocksRatio < threshold {
 				continue
 			}
 			overlaps := p.vers.Overlaps(lastNonEmptyLevel, f.UserKeyBounds())
diff --git a/options.go b/options.go
index cf70fa8f..e6874357 100644
--- a/options.go
+++ b/options.go
@@ -648,17 +648,19 @@ type Options struct {
 		// sstable writers. The default value is 0.5.
 		DeletionSizeRatioThreshold float32
 
-		// TombstoneDenseCompactionThreshold is the minimum percent of data
-		// blocks in a table that must be tombstone-dense for that table to be
-		// eligible for a tombstone density compaction. It should be defined as a
-		// ratio out of 1. The default value is 0.10.
+		// TombstoneDenseCompactionThreshold is a function that returns the minimum
+		// percent of data blocks in a table that must be tombstone-dense for that
+		// table to be eligible for a tombstone density compaction. The value should
+		// be defined as a ratio out of 1. The default value is 0.10.
 		//
 		// If multiple tables are eligible for a tombstone density compaction, then
 		// tables with a higher percent of tombstone-dense blocks are still
 		// prioritized for compaction.
 		//
-		// A zero or negative value disables tombstone density compactions.
-		TombstoneDenseCompactionThreshold float64
+		// Using a function allows for dynamic reconfiguration of the threshold based
+		// on workload characteristics. A zero or negative value disables tombstone
+		// density compactions.
+		TombstoneDenseCompactionThreshold func() float64
 
 		// FileCacheShards is the number of shards per file cache.
 		// Reducing the value can reduce the number of idle goroutines per DB
@@ -1516,8 +1518,8 @@ func (o *Options) EnsureDefaults() {
 	if o.Experimental.EnableColumnarBlocks == nil {
 		o.Experimental.EnableColumnarBlocks = func() bool { return true }
 	}
-	if o.Experimental.TombstoneDenseCompactionThreshold == 0 {
-		o.Experimental.TombstoneDenseCompactionThreshold = 0.10
+	if o.Experimental.TombstoneDenseCompactionThreshold == nil {
+		o.Experimental.TombstoneDenseCompactionThreshold = func() float64 { return 0.10 }
 	}
 	if o.Experimental.FileCacheShards <= 0 {
 		o.Experimental.FileCacheShards = runtime.GOMAXPROCS(0)
@@ -1662,7 +1664,7 @@ func (o *Options) String() string {
 	fmt.Fprintf(&buf, "  read_sampling_multiplier=%d\n", o.Experimental.ReadSamplingMultiplier)
 	fmt.Fprintf(&buf, "  num_deletions_threshold=%d\n", o.Experimental.NumDeletionsThreshold)
 	fmt.Fprintf(&buf, "  deletion_size_ratio_threshold=%f\n", o.Experimental.DeletionSizeRatioThreshold)
-	fmt.Fprintf(&buf, "  tombstone_dense_compaction_threshold=%f\n", o.Experimental.TombstoneDenseCompactionThreshold)
+	fmt.Fprintf(&buf, "  tombstone_dense_compaction_threshold=%f\n", o.Experimental.TombstoneDenseCompactionThreshold())
 	// We no longer care about strict_wal_tail, but set it to true in case an
 	// older version reads the options.
 	fmt.Fprintf(&buf, "  strict_wal_tail=%t\n", true)
@@ -2087,7 +2089,11 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
 				o.Experimental.DeletionSizeRatioThreshold = float32(val)
 				err = parseErr
 			case "tombstone_dense_compaction_threshold":
-				o.Experimental.TombstoneDenseCompactionThreshold, err = strconv.ParseFloat(value, 64)
+				var threshold float64
+				threshold, err = strconv.ParseFloat(value, 64)
+				if err == nil {
+					o.Experimental.TombstoneDenseCompactionThreshold = func() float64 { return threshold }
+				}
 			case "table_cache_shards":
 				o.Experimental.FileCacheShards, err = strconv.Atoi(value)
 			case "table_format":



Test patch:
diff --git a/compaction_test.go b/compaction_test.go
index 796ee342..da5c7383 100644
--- a/compaction_test.go
+++ b/compaction_test.go
@@ -3242,7 +3242,7 @@ func TestTombstoneDensityCompactionMoveOptimization(t *testing.T) {
 	)
 
 	opts := DefaultOptions()
-	opts.Experimental.TombstoneDenseCompactionThreshold = 0.5 // Lower for test
+	opts.Experimental.TombstoneDenseCompactionThreshold = func() float64 { return 0.5 } // Lower for test
 	opts.Experimental.NumDeletionsThreshold = 1
 	opts.Experimental.EnableColumnarBlocks = func() bool { return true }
 	opts.Experimental.CompactionScheduler = NewConcurrencyLimitSchedulerWithNoPeriodicGrantingForTest()
@@ -3340,7 +3340,7 @@ func TestTombstoneDensityCompactionMoveOptimization_NoMoveWithOverlap(t *testing
 	)
 
 	opts := DefaultOptions()
-	opts.Experimental.TombstoneDenseCompactionThreshold = 0.5 // Lower for test
+	opts.Experimental.TombstoneDenseCompactionThreshold = func() float64 { return 0.5 } // Lower for test
 	opts.Experimental.NumDeletionsThreshold = 1
 	opts.Experimental.EnableColumnarBlocks = func() bool { return true }
 	opts.Experimental.CompactionScheduler = NewConcurrencyLimitSchedulerWithNoPeriodicGrantingForTest()
@@ -3419,7 +3419,7 @@ func TestTombstoneDensityCompactionMoveOptimization_GrandparentOverlapTooLarge(t
 	)
 
 	opts := DefaultOptions()
-	opts.Experimental.TombstoneDenseCompactionThreshold = 0.5
+	opts.Experimental.TombstoneDenseCompactionThreshold = func() float64 { return 0.5 }
 	opts.Experimental.NumDeletionsThreshold = 1
 	opts.Experimental.EnableColumnarBlocks = func() bool { return true }
 	opts.Experimental.CompactionScheduler = NewConcurrencyLimitSchedulerWithNoPeriodicGrantingForTest()
@@ -3481,7 +3481,7 @@ func TestTombstoneDensityCompactionMoveOptimization_BelowDensityThreshold(t *tes
 	)
 
 	opts := DefaultOptions()
-	opts.Experimental.TombstoneDenseCompactionThreshold = 0.9 // Set high threshold
+	opts.Experimental.TombstoneDenseCompactionThreshold = func() float64 { return 0.9 } // Set high threshold
 	opts.Experimental.NumDeletionsThreshold = 1
 	opts.Experimental.EnableColumnarBlocks = func() bool { return true }
 	opts.Experimental.CompactionScheduler = NewConcurrencyLimitSchedulerWithNoPeriodicGrantingForTest()
@@ -3528,7 +3528,7 @@ func TestTombstoneDensityCompactionMoveOptimization_InvalidStats(t *testing.T) {
 	)
 
 	opts := DefaultOptions()
-	opts.Experimental.TombstoneDenseCompactionThreshold = 0.5
+	opts.Experimental.TombstoneDenseCompactionThreshold = func() float64 { return 0.5 }
 	opts.Experimental.NumDeletionsThreshold = 1
 	opts.Experimental.EnableColumnarBlocks = func() bool { return true }
 	opts.Experimental.CompactionScheduler = NewConcurrencyLimitSchedulerWithNoPeriodicGrantingForTest()
diff --git a/metamorphic/options_test.go b/metamorphic/options_test.go
index 5fcabc99..3b66f6b3 100644
--- a/metamorphic/options_test.go
+++ b/metamorphic/options_test.go
@@ -85,6 +85,7 @@ func TestOptionsRoundtrip(t *testing.T) {
 		"Experimental.RemoteStorage:",
 		"Experimental.SingleDeleteInvariantViolationCallback:",
 		"Experimental.EnableDeleteOnlyCompactionExcises:",
+		"Experimental.TombstoneDenseCompactionThreshold:",
 		"Experimental.ValueSeparationPolicy:",
 		"Levels[0].Compression:",
 		"Levels[1].Compression:",
@@ -115,6 +116,7 @@ func TestOptionsRoundtrip(t *testing.T) {
 		expectEqualFn(t, o.Opts.Experimental.DisableIngestAsFlushable, parsed.Opts.Experimental.DisableIngestAsFlushable)
 		expectEqualFn(t, o.Opts.Experimental.IngestSplit, parsed.Opts.Experimental.IngestSplit)
 		expectEqualFn(t, o.Opts.Experimental.CompactionGarbageFractionForMaxConcurrency, parsed.Opts.Experimental.CompactionGarbageFractionForMaxConcurrency)
+		expectEqualFn(t, o.Opts.Experimental.TombstoneDenseCompactionThreshold, parsed.Opts.Experimental.TombstoneDenseCompactionThreshold)
 		expectEqualFn(t, o.Opts.Experimental.ValueSeparationPolicy, parsed.Opts.Experimental.ValueSeparationPolicy)
 		expectEqualFn(t, o.Opts.TargetByteDeletionRate, parsed.Opts.TargetByteDeletionRate)
 
diff --git a/options_test.go b/options_test.go
index d685618b..d8ad7aae 100644
--- a/options_test.go
+++ b/options_test.go
@@ -404,7 +404,7 @@ func TestOptionsParse(t *testing.T) {
 			opts.Experimental.ReadSamplingMultiplier = 400
 			opts.Experimental.NumDeletionsThreshold = 500
 			opts.Experimental.DeletionSizeRatioThreshold = 0.7
-			opts.Experimental.TombstoneDenseCompactionThreshold = 0.2
+			opts.Experimental.TombstoneDenseCompactionThreshold = func() float64 { return 0.2 }
 			opts.Experimental.FileCacheShards = 500
 			opts.Experimental.SecondaryCacheSizeBytes = 1024
 			opts.Experimental.ValueSeparationPolicy = func() ValueSeparationPolicy {
diff --git a/testdata/compaction/value_separation b/testdata/compaction/value_separation
index d9e539be..94653155 100644
--- a/testdata/compaction/value_separation
+++ b/testdata/compaction/value_separation
@@ -123,14 +123,14 @@ metrics
       |                             |                |       |   ingested   |     moved    |    written   |       |    amp   |    val sep    |     multilevel
 level | tables  size val-bl vtables | score  ff  cff |   in  | tables  size | tables  size | tables  size |  read |   r   w  | refsz  valblk |   top   in  read
 ------+-----------------------------+----------------+-------+--------------+--------------+--------------+-------+----------+---------------+------------------
-    0 |     0     0B     0B       0 |    -    0    0 |   41B |     0     0B |     0     0B |     1   861B |    0B |   0 25.9 |    0B      0B |    0B    0B    0B
+    0 |     0     0B     0B       0 |    -    0    0 |   41B |     0     0B |     0     0B |     1   861B |    0B |   0 23.4 |    0B      0B |    0B    0B    0B
     1 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     1   861B |     0     0B |    0B |   0    0 |    0B      0B |    0B    0B    0B
     2 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B |    0B    0B    0B
     3 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     1   861B |     0     0B |    0B |   0    0 |    0B      0B |    0B    0B    0B
     4 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B |    0B    0B    0B
     5 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     1   861B |     0     0B |    0B |   0    0 |    0B      0B |    0B    0B    0B
     6 |     1   931B     0B       0 |    - 0.00 0.00 |  861B |     0     0B |     0     0B |     1   931B | 1.7KB |   1 1.21 |  112B      0B |    0B    0B    0B
-total |     1   931B     0B       0 |    -    -    - |   41B |     0     0B |     3  2.5KB |     2  1.8KB | 1.7KB |   1 52.3 |  112B      0B |    0B    0B    0B
+total |     1   931B     0B       0 |    -    -    - |   41B |     0     0B |     3  2.5KB |     2  1.8KB | 1.7KB |   1 49.9 |  112B      0B |    0B    0B    0B
 ----------------------------------------------------------------------------------------------------------------------------------------------------------------
 WAL: 1 files (0B)  in: 30B  written: 41B (37% overhead)
 Flushes: 1
@@ -363,14 +363,14 @@ metrics
       |                             |                |       |   ingested   |     moved    |    written   |       |    amp   |    val sep
 level | tables  size val-bl vtables | score  ff  cff |   in  | tables  size | tables  size | tables  size |  read |   r   w  | refsz  valblk
 ------+-----------------------------+----------------+-------+--------------+--------------+--------------+-------+----------+--------------
-    0 |     0     0B     0B       0 |    -    0    0 |  156B |     0     0B |     0     0B |     2  1.8KB |    0B |   0 14.8 |    0B      0B
+    0 |     0     0B     0B       0 |    -    0    0 |  156B |     0     0B |     0     0B |     2  1.8KB |    0B |   0 13.2 |    0B      0B
     1 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     2 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     3 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     4 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     5 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     6 |     1   932B     0B       0 |    - 0.00 0.00 | 1.8KB |     0     0B |     0     0B |     1   932B | 1.8KB |   1 0.51 |  232B      0B
-total |     1   932B     0B       0 |    -    -    - |  156B |     0     0B |     0     0B |     3  2.8KB | 1.8KB |   1 21.8 |  232B      0B
+total |     1   932B     0B       0 |    -    -    - |  156B |     0     0B |     0     0B |     3  2.8KB | 1.8KB |   1 20.2 |  232B      0B
 --------------------------------------------------------------------------------------------------------------------------------------------
 WAL: 1 files (0B)  in: 126B  written: 156B (24% overhead)
 Flushes: 2
diff --git a/testdata/metrics b/testdata/metrics
index 82c94b70..9bf8ec2e 100644
--- a/testdata/metrics
+++ b/testdata/metrics
@@ -350,14 +350,14 @@ metrics
       |                             |                |       |   ingested   |     moved    |    written   |       |    amp   |    val sep
 level | tables  size val-bl vtables | score  ff  cff |   in  | tables  size | tables  size | tables  size |  read |   r   w  | refsz  valblk
 ------+-----------------------------+----------------+-------+--------------+--------------+--------------+-------+----------+--------------
-    0 |     7  6.0KB     0B       0 |    - 0.25 0.25 |  165B |     0     0B |     0     0B |     9  7.5KB |    0B |   1 54.2 |  644B      0B
+    0 |     7  6.0KB     0B       0 |    - 0.25 0.25 |  165B |     0     0B |     0     0B |     9  7.5KB |    0B |   1 50.3 |  644B      0B
     1 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     2 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     3 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     4 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     5 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     6 |     2  1.5KB     0B       0 |    - 0.00 0.00 | 1.5KB |     0     0B |     0     0B |     2  1.5KB | 1.5KB |   1 0.99 |    0B      0B
-total |     9  7.5KB     0B       0 |    -    -    - |  165B |     0     0B |     0     0B |    11  9.1KB | 1.5KB |   2 64.4 |  644B      0B
+total |     9  7.5KB     0B       0 |    -    -    - |  165B |     0     0B |     0     0B |    11  9.1KB | 1.5KB |   2 60.5 |  644B      0B
 --------------------------------------------------------------------------------------------------------------------------------------------
 WAL: 1 files (0B)  in: 116B  written: 165B (42% overhead)
 Flushes: 3
@@ -422,14 +422,14 @@ metrics
       |                             |                |       |   ingested   |     moved    |    written   |       |    amp   |    val sep
 level | tables  size val-bl vtables | score  ff  cff |   in  | tables  size | tables  size | tables  size |  read |   r   w  | refsz  valblk
 ------+-----------------------------+----------------+-------+--------------+--------------+--------------+-------+----------+--------------
-    0 |     0     0B     0B       0 |    -    0    0 |  165B |     0     0B |     0     0B |     9  7.5KB |    0B |   0 54.2 |    0B      0B
+    0 |     0     0B     0B       0 |    -    0    0 |  165B |     0     0B |     0     0B |     9  7.5KB |    0B |   0 50.3 |    0B      0B
     1 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     2 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     3 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     4 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     5 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     6 |     9  7.4KB     0B       0 |    - 0.00 0.00 | 7.5KB |     0     0B |     0     0B |     9  7.4KB | 7.5KB |   1 1.00 |  644B      0B
-total |     9  7.4KB     0B       0 |    -    -    - |  165B |     0     0B |     0     0B |    18   15KB | 7.5KB |   1  101 |  644B      0B
+total |     9  7.4KB     0B       0 |    -    -    - |  165B |     0     0B |     0     0B |    18   15KB | 7.5KB |   1 97.5 |  644B      0B
 --------------------------------------------------------------------------------------------------------------------------------------------
 WAL: 1 files (0B)  in: 116B  written: 165B (42% overhead)
 Flushes: 3
@@ -545,14 +545,14 @@ metrics
       |                             |                |       |   ingested   |     moved    |    written   |       |    amp   |    val sep
 level | tables  size val-bl vtables | score  ff  cff |   in  | tables  size | tables  size | tables  size |  read |   r   w  | refsz  valblk
 ------+-----------------------------+----------------+-------+--------------+--------------+--------------+-------+----------+--------------
-    0 |     6  4.5KB     0B       0 |    - 0.50 0.50 |  211B |     3  2.2KB |     0     0B |    12  9.7KB |    0B |   2 53.3 |    0B      0B
+    0 |     6  4.5KB     0B       0 |    - 0.50 0.50 |  211B |     3  2.2KB |     0     0B |    12  9.7KB |    0B |   2 50.2 |    0B      0B
     1 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     2 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     3 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     4 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     5 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     6 |     9  7.4KB     0B       0 |    - 0.00 0.00 | 7.5KB |     0     0B |     0     0B |     9  7.4KB | 7.5KB |   1 1.00 |  644B      0B
-total |    15   12KB     0B       0 |    -    -    - | 2.4KB |     3  2.2KB |     0     0B |    21   20KB | 7.5KB |   3 8.53 |  644B      0B
+total |    15   12KB     0B       0 |    -    -    - | 2.4KB |     3  2.2KB |     0     0B |    21   20KB | 7.5KB |   3 8.27 |  644B      0B
 --------------------------------------------------------------------------------------------------------------------------------------------
 WAL: 1 files (0B)  in: 176B  written: 211B (20% overhead)
 Flushes: 8
@@ -630,14 +630,14 @@ metrics
       |                             |                |       |   ingested   |     moved    |    written   |       |    amp   |    val sep
 level | tables  size val-bl vtables | score  ff  cff |   in  | tables  size | tables  size | tables  size |  read |   r   w  | refsz  valblk
 ------+-----------------------------+----------------+-------+--------------+--------------+--------------+-------+----------+--------------
-    0 |    13  9.8KB     0B       0 |    - 0.50 0.50 |  277B |     3  2.2KB |     0     0B |    19   15KB |    0B |   2 60.0 |    0B      0B
+    0 |    13  9.8KB     0B       0 |    - 0.50 0.50 |  277B |     3  2.2KB |     0     0B |    19   15KB |    0B |   2 57.7 |    0B      0B
     1 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     2 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     3 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     4 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     5 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     6 |     9  7.4KB     0B       0 |    - 0.00 0.00 | 7.5KB |     0     0B |     0     0B |     9  7.4KB | 7.5KB |   1 1.00 |  644B      0B
-total |    22   17KB     0B       0 |    -    -    - | 2.5KB |     3  2.2KB |     0     0B |    28   25KB | 7.5KB |   3 10.4 |  644B      0B
+total |    22   17KB     0B       0 |    -    -    - | 2.5KB |     3  2.2KB |     0     0B |    28   25KB | 7.5KB |   3 10.2 |  644B      0B
 --------------------------------------------------------------------------------------------------------------------------------------------
 WAL: 1 files (0B)  in: 223B  written: 277B (24% overhead)
 Flushes: 9
@@ -727,14 +727,14 @@ metrics zero-cache-hits-misses
       |                             |                |       |   ingested   |     moved    |    written   |       |    amp   |    val sep
 level | tables  size val-bl vtables | score  ff  cff |   in  | tables  size | tables  size | tables  size |  read |   r   w  | refsz  valblk
 ------+-----------------------------+----------------+-------+--------------+--------------+--------------+-------+----------+--------------
-    0 |    11  8.2KB     0B       0 |    - 0.50 0.50 |  277B |     3  2.2KB |     0     0B |    19   15KB |    0B |   2 60.0 |    0B      0B
+    0 |    11  8.2KB     0B       0 |    - 0.50 0.50 |  277B |     3  2.2KB |     0     0B |    19   15KB |    0B |   2 57.7 |    0B      0B
     1 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     2 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     3 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     4 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     5 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     6 |    10  8.2KB     0B       0 |    - 0.00 0.00 | 7.5KB |     1   765B |     0     0B |     9  7.4KB | 7.5KB |   1 1.00 |  644B      0B
-total |    21   16KB     0B       0 |    -    -    - | 3.3KB |     4  3.0KB |     0     0B |    28   26KB | 7.5KB |   3 8.27 |  644B      0B
+total |    21   16KB     0B       0 |    -    -    - | 3.3KB |     4  3.0KB |     0     0B |    28   26KB | 7.5KB |   3 8.08 |  644B      0B
 --------------------------------------------------------------------------------------------------------------------------------------------
 WAL: 1 files (0B)  in: 223B  written: 277B (24% overhead)
 Flushes: 9
@@ -868,14 +868,14 @@ metrics zero-cache-hits-misses
       |                             |                |       |   ingested   |     moved    |    written   |       |    amp   |    val sep
 level | tables  size val-bl vtables | score  ff  cff |   in  | tables  size | tables  size | tables  size |  read |   r   w  | refsz  valblk
 ------+-----------------------------+----------------+-------+--------------+--------------+--------------+-------+----------+--------------
-    0 |     0     0B     0B       0 |    -    0    0 |  277B |     3  2.2KB |     0     0B |    19   15KB |    0B |   0 60.0 |    0B      0B
+    0 |     0     0B     0B       0 |    -    0    0 |  277B |     3  2.2KB |     0     0B |    19   15KB |    0B |   0 57.7 |    0B      0B
     1 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     2 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     3 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     4 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     5 |     0     0B     0B       0 |    -    0    0 |    0B |     0     0B |     0     0B |     0     0B |    0B |   0    0 |    0B      0B
     6 |    18   14KB     0B       0 |    - 0.00 0.00 |  15KB |     2  1.5KB |     0     0B |    16   13KB |  15KB |   1 0.85 |  644B      0B
-total |    18   14KB     0B       0 |    -    -    - | 4.0KB |     5  3.7KB |     0     0B |    35   32KB |  15KB |   1 8.23 |  644B      0B
+total |    18   14KB     0B       0 |    -    -    - | 4.0KB |     5  3.7KB |     0     0B |    35   32KB |  15KB |   1 8.07 |  644B      0B
 --------------------------------------------------------------------------------------------------------------------------------------------
 WAL: 1 files (0B)  in: 223B  written: 277B (24% overhead)
 Flushes: 9