Repository: petermattis/pebble
Pull request #134 (2019-05-10 21:18:38)

Adjust dynamic-base-level heuristic
Previously, the target size of the base level was computed as
`max(L1MaxBytes,L0Size)`. This had the negative effect of allowing the
target size for the base level to grow dramatically if L0 was growing
dramatically, which would prevent the dynamic-base-level heuristic from
adding additional levels to the LSM.

Rename `Options.L1MaxBytes` to `Options.LBaseMaxBytes`.

petermattis: 

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.png" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/petermattis/pebble/134)
<!-- Reviewable:end -->


Patch:
diff --git a/compaction_picker.go b/compaction_picker.go
index 52de0e0..e07abce 100644
--- a/compaction_picker.go
+++ b/compaction_picker.go
@@ -78,11 +78,7 @@ func (p *compactionPicker) initLevelMaxBytes(v *version, opts *db.Options) {
 
 	levelMultiplier := 10.0
 
-	l0Size := int64(totalSize(v.files[0]))
-	baseBytesMax := opts.L1MaxBytes
-	if baseBytesMax < l0Size {
-		baseBytesMax = l0Size
-	}
+	baseBytesMax := opts.LBaseMaxBytes
 	baseBytesMin := int64(float64(baseBytesMax) / levelMultiplier)
 
 	curLevelSize := maxLevelSize
@@ -111,27 +107,6 @@ func (p *compactionPicker) initLevelMaxBytes(v *version, opts *db.Options) {
 		}
 	}
 
-	if l0Size > baseLevelSize &&
-		(l0Size > opts.L1MaxBytes ||
-			(len(v.files)/2) >= opts.L0CompactionThreshold) {
-		// We adjust the base level according to actual L0 size, and adjust the
-		// level multiplier accordingly, when:
-		//
-		//   1. the L0 size is larger than level size base, or
-		//   2. number of L0 files reaches twice the L0->L1 compaction threshold
-		//
-		// We don't do this otherwise to keep the LSM-tree structure stable unless
-		// the L0 compaction is backlogged.
-		baseLevelSize = l0Size
-		if p.baseLevel == numLevels-1 {
-			levelMultiplier = 1.0
-		} else {
-			levelMultiplier = math.Pow(
-				float64(maxLevelSize)/float64(baseLevelSize),
-				1.0/float64(numLevels-p.baseLevel-1))
-		}
-	}
-
 	levelSize := baseLevelSize
 	for level := p.baseLevel; level < numLevels; level++ {
 		if level > p.baseLevel {
diff --git a/db/options.go b/db/options.go
index 662be85..dcaae82 100644
--- a/db/options.go
+++ b/db/options.go
@@ -234,10 +234,12 @@ type Options struct {
 	// threshold is reached.
 	L0StopWritesThreshold int
 
-	// The maximum number of bytes for L1. The maximum number of bytes for other
-	// levels is computed dynamically based on this value. When the maximum
-	// number of bytes for a level is exceeded, compaction is requested.
-	L1MaxBytes int64
+	// The maximum number of bytes for LBase. The base level is the level which
+	// L0 is compacted into. The base level is determined dynamically based on
+	// the existing data in the LSM. The maximum number of bytes for other levels
+	// is computed dynamically based on the base level's maximum size. When the
+	// maximum number of bytes for a level is exceeded, compaction is requested.
+	LBaseMaxBytes int64
 
 	// Per-level options. Options for at least one level must be specified. The
 	// options for the last level are used for all subsequent levels.
@@ -314,8 +316,8 @@ func (o *Options) EnsureDefaults() *Options {
 	if o.L0StopWritesThreshold <= 0 {
 		o.L0StopWritesThreshold = 12
 	}
-	if o.L1MaxBytes <= 0 {
-		o.L1MaxBytes = 64 << 20 // 64 MB
+	if o.LBaseMaxBytes <= 0 {
+		o.LBaseMaxBytes = 64 << 20 // 64 MB
 	}
 	if o.Levels == nil {
 		o.Levels = make([]LevelOptions, 1)
@@ -384,7 +386,7 @@ func (o *Options) String() string {
 	fmt.Fprintf(&buf, "  l0_compaction_threshold=%d\n", o.L0CompactionThreshold)
 	fmt.Fprintf(&buf, "  l0_slowdown_writes_threshold=%d\n", o.L0SlowdownWritesThreshold)
 	fmt.Fprintf(&buf, "  l0_stop_writes_threshold=%d\n", o.L0StopWritesThreshold)
-	fmt.Fprintf(&buf, "  l1_max_bytes=%d\n", o.L1MaxBytes)
+	fmt.Fprintf(&buf, "  lbase_max_bytes=%d\n", o.LBaseMaxBytes)
 	fmt.Fprintf(&buf, "  max_manifest_file_size=%d\n", o.MaxManifestFileSize)
 	fmt.Fprintf(&buf, "  max_open_files=%d\n", o.MaxOpenFiles)
 	fmt.Fprintf(&buf, "  mem_table_size=%d\n", o.MemTableSize)


Test patch:
diff --git a/cmd/pebble/test.go b/cmd/pebble/test.go
index 7405e50..651448c 100644
--- a/cmd/pebble/test.go
+++ b/cmd/pebble/test.go
@@ -199,7 +199,7 @@ func runTest(dir string, t test) {
 		L0CompactionThreshold:       2,
 		L0SlowdownWritesThreshold:   20,
 		L0StopWritesThreshold:       32,
-		L1MaxBytes:                  64 << 20, // 64 MB
+		LBaseMaxBytes:               64 << 20, // 64 MB
 		Levels: []db.LevelOptions{{
 			BlockSize: 32 << 10,
 		}},
diff --git a/compaction_picker_test.go b/compaction_picker_test.go
index ee0de13..81f1c58 100644
--- a/compaction_picker_test.go
+++ b/compaction_picker_test.go
@@ -27,7 +27,7 @@ func TestCompactionPickerLevelMaxBytes(t *testing.T) {
 					return fmt.Sprintf("%s expects 1 argument", d.Cmd)
 				}
 				var err error
-				opts.L1MaxBytes, err = strconv.ParseInt(d.CmdArgs[0].Key, 10, 64)
+				opts.LBaseMaxBytes, err = strconv.ParseInt(d.CmdArgs[0].Key, 10, 64)
 				if err != nil {
 					return err.Error()
 				}
@@ -89,7 +89,7 @@ func TestCompactionPickerTargetLevel(t *testing.T) {
 					return fmt.Sprintf("%s expects 1 argument", d.Cmd)
 				}
 				var err error
-				opts.L1MaxBytes, err = strconv.ParseInt(d.CmdArgs[0].Key, 10, 64)
+				opts.LBaseMaxBytes, err = strconv.ParseInt(d.CmdArgs[0].Key, 10, 64)
 				if err != nil {
 					return err.Error()
 				}
diff --git a/db/options_test.go b/db/options_test.go
index 5bf0f81..bd07a82 100644
--- a/db/options_test.go
+++ b/db/options_test.go
@@ -47,7 +47,7 @@ func TestOptionsString(t *testing.T) {
   l0_compaction_threshold=4
   l0_slowdown_writes_threshold=8
   l0_stop_writes_threshold=12
-  l1_max_bytes=67108864
+  lbase_max_bytes=67108864
   max_manifest_file_size=134217728
   max_open_files=1000
   mem_table_size=4194304
diff --git a/testdata/compaction_picker_level_max_bytes b/testdata/compaction_picker_level_max_bytes
index 5434995..a6b74f4 100644
--- a/testdata/compaction_picker_level_max_bytes
+++ b/testdata/compaction_picker_level_max_bytes
@@ -124,5 +124,5 @@ init 1
 0: 4
 6: 10
 ----
-5: 4
+5: 1
 6: 10
diff --git a/testdata/compaction_picker_target_level b/testdata/compaction_picker_target_level
index 65d2fda..ab0df71 100644
--- a/testdata/compaction_picker_target_level
+++ b/testdata/compaction_picker_target_level
@@ -73,4 +73,4 @@ pick 1
 5: 2
 6: 10
 ----
-0: 1.2
+5: 2.0