Repository: cockroachdb/pebble
Pull request #1417 (2021-12-16 22:59:14)

internal/metamorphic: generate keys with suffixes
Generate keys with human-readable suffixes like '`@3`'. The key manager is
extended to track unique in-use prefixes as well. A fixed portion of new keys
are generated using an existing prefix but a new suffix.

New keys' suffixes are generated using a skewed-latest distribution, mimicking
MVCC timestamps

cockroach-teamcity: 

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.png" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/cockroachdb/pebble/1417)
<!-- Reviewable:end -->


Patch:
diff --git a/internal/metamorphic/config.go b/internal/metamorphic/config.go
index 36c83522..5f9d9345 100644
--- a/internal/metamorphic/config.go
+++ b/internal/metamorphic/config.go
@@ -4,6 +4,8 @@
 
 package metamorphic
 
+import "github.com/cockroachdb/pebble/internal/randvar"
+
 type opType int
 
 const (
@@ -48,6 +50,15 @@ type config struct {
 	// weight for opType(i).
 	ops []int
 
+	// newPrefix configures the probability that when generating a new user key,
+	// the generated key uses a new key prefix rather than an existing prefix
+	// with a suffix.
+	newPrefix float64
+	// suffixDist defines the distribution of key suffixes. It's a dynamic
+	// randvar to roughly emulate workloads with MVCC timestamps, skewing
+	// towards most recent timestamps.
+	suffixDist randvar.Dynamic
+
 	// TODO(peter): unimplemented
 	// keyDist        randvar.Dynamic
 	// keySizeDist    randvar.Static
@@ -57,41 +68,57 @@ type config struct {
 	// upperBoundFrac float64
 }
 
-var defaultConfig = config{
-	// dbClose is not in this list since it is deterministically generated once, at the end of the test.
-	ops: []int{
-		batchAbort:          5,
-		batchCommit:         5,
-		dbCheckpoint:        1,
-		dbCompact:           1,
-		dbFlush:             2,
-		dbRestart:           2,
-		iterClose:           5,
-		iterFirst:           100,
-		iterLast:            100,
-		iterNext:            100,
-		iterNextWithLimit:   20,
-		iterPrev:            100,
-		iterPrevWithLimit:   20,
-		iterSeekGE:          100,
-		iterSeekGEWithLimit: 20,
-		iterSeekLT:          100,
-		iterSeekLTWithLimit: 20,
-		iterSeekPrefixGE:    100,
-		iterSetBounds:       100,
-		newBatch:            5,
-		newIndexedBatch:     5,
-		newIter:             10,
-		newIterUsingClone:   5,
-		newSnapshot:         10,
-		readerGet:           100,
-		snapshotClose:       10,
-		writerApply:         10,
-		writerDelete:        100,
-		writerDeleteRange:   50,
-		writerIngest:        100,
-		writerMerge:         100,
-		writerSet:           100,
-		writerSingleDelete:  50,
-	},
+func defaultConfig() config {
+	return config{
+		// dbClose is not in this list since it is deterministically generated once, at the end of the test.
+		ops: []int{
+			batchAbort:          5,
+			batchCommit:         5,
+			dbCheckpoint:        1,
+			dbCompact:           1,
+			dbFlush:             2,
+			dbRestart:           2,
+			iterClose:           5,
+			iterFirst:           100,
+			iterLast:            100,
+			iterNext:            100,
+			iterNextWithLimit:   20,
+			iterPrev:            100,
+			iterPrevWithLimit:   20,
+			iterSeekGE:          100,
+			iterSeekGEWithLimit: 20,
+			iterSeekLT:          100,
+			iterSeekLTWithLimit: 20,
+			iterSeekPrefixGE:    100,
+			iterSetBounds:       100,
+			newBatch:            5,
+			newIndexedBatch:     5,
+			newIter:             10,
+			newIterUsingClone:   5,
+			newSnapshot:         10,
+			readerGet:           100,
+			snapshotClose:       10,
+			writerApply:         10,
+			writerDelete:        100,
+			writerDeleteRange:   50,
+			writerIngest:        100,
+			writerMerge:         100,
+			writerSet:           100,
+			writerSingleDelete:  50,
+		},
+		// Use a new prefix 75% of the time (and 25% of the time use an existing
+		// prefix with an alternative suffix).
+		newPrefix: 0.75,
+		// Use a skewed distribution of suffixes to mimic MVCC timestamps. The
+		// range will be widened whenever a suffix is found to already be in use
+		// for a particular prefix.
+		suffixDist: mustDynamic(randvar.NewSkewedLatest(0, 1, 0.99)),
+	}
+}
+
+func mustDynamic(dyn randvar.Dynamic, err error) randvar.Dynamic {
+	if err != nil {
+		panic(err)
+	}
+	return dyn
 }
diff --git a/internal/metamorphic/generator.go b/internal/metamorphic/generator.go
index 7ee98fd1..edcb6ab6 100644
--- a/internal/metamorphic/generator.go
+++ b/internal/metamorphic/generator.go
@@ -9,6 +9,7 @@ import (
 	"fmt"
 
 	"github.com/cockroachdb/pebble/internal/randvar"
+	"github.com/cockroachdb/pebble/internal/testkeys"
 	"golang.org/x/exp/rand"
 )
 
@@ -18,6 +19,7 @@ type iterBounds struct {
 }
 
 type generator struct {
+	cfg config
 	rng *rand.Rand
 
 	init *initOp
@@ -61,8 +63,9 @@ type generator struct {
 	snapshots map[objID]objIDSet
 }
 
-func newGenerator(rng *rand.Rand) *generator {
+func newGenerator(rng *rand.Rand, cfg config) *generator {
 	g := &generator{
+		cfg:         cfg,
 		rng:         rng,
 		init:        &initOp{},
 		keyManager:  newKeyManager(),
@@ -79,7 +82,7 @@ func newGenerator(rng *rand.Rand) *generator {
 }
 
 func generate(rng *rand.Rand, count uint64, cfg config) []op {
-	g := newGenerator(rng)
+	g := newGenerator(rng, cfg)
 
 	generators := []func(){
 		batchAbort:          g.batchAbort,
@@ -156,20 +159,96 @@ func (g *generator) randKeyToRead(newKey float64) []byte {
 }
 
 func (g *generator) randKeyHelper(keys [][]byte, newKey float64) []byte {
-	if n := len(keys); n > 0 && g.rng.Float64() > newKey {
-		return keys[g.rng.Intn(n)]
-	}
-	for {
-		key := g.randValue(4, 12)
-		if g.keyManager.addNewKey(key) {
-			return key
+	switch {
+	case len(keys) > 0 && g.rng.Float64() > newKey:
+		// Use an existing user key.
+		return keys[g.rng.Intn(len(keys))]
+
+	case len(keys) > 0 && g.rng.Float64() > g.cfg.newPrefix:
+		// Use an existing prefix but a new suffix, producing a new user key.
+		prefixes := g.keyManager.prefixes()
+		var key []byte
+		for {
+			// Pick a prefix on each iteration in case most or all suffixes are
+			// already in use for any individual prefix.
+			p := g.rng.Intn(len(prefixes))
+			suffix := int(g.cfg.suffixDist.Uint64(g.rng))
+
+			if suffix > 0 {
+				key = resizeBuffer(key, len(prefixes[p]), testkeys.SuffixLen(suffix))
+				n := copy(key, prefixes[p])
+				testkeys.WriteSuffix(key[n:], suffix)
+			} else {
+				key = resizeBuffer(key, len(prefixes[p]), 0)
+				copy(key, prefixes[p])
+			}
+			if g.keyManager.addNewKey(key) {
+				return key
+			}
+
+			// If the generated key already existed, increase the suffix
+			// distribution to make a generating a new user key with an existing
+			// prefix more likely.
+			g.cfg.suffixDist.IncMax(1)
+		}
+
+	default:
+		// Use a new prefix, producing a new user key.
+		suffix := int(g.cfg.suffixDist.Uint64(g.rng))
+		var key []byte
+		for {
+			key = g.randKeyHelperSuffix(nil, 4, 12, suffix)
+			if !g.keyManager.prefixExists(key[:testkeys.Comparer.Split(key)]) {
+				if !g.keyManager.addNewKey(key) {
+					panic("key must not exist if prefix doesn't exist")
+				}
+				return key
+			}
 		}
 	}
 }
 
+// randKeyHelperSuffix is a helper function for randKeyHelper, and should not be
+// invoked directly.
+func (g *generator) randKeyHelperSuffix(dst []byte, minPrefixLen, maxPrefixLen, suffix int) []byte {
+	n := minPrefixLen
+	if maxPrefixLen > minPrefixLen {
+		n += g.rng.Intn(maxPrefixLen - minPrefixLen)
+	}
+	// In order to test a mix of suffixed and unsuffixed keys, omit the zero
+	// suffix.
+	if suffix == 0 {
+		dst = resizeBuffer(dst, n, 0)
+		g.fillRand(dst)
+		return dst
+	}
+	suffixLen := testkeys.SuffixLen(suffix)
+	dst = resizeBuffer(dst, n, suffixLen)
+	g.fillRand(dst[:n])
+	testkeys.WriteSuffix(dst[n:], suffix)
+	return dst
+}
+
+func resizeBuffer(buf []byte, prefixLen, suffixLen int) []byte {
+	if cap(buf) >= prefixLen+suffixLen {
+		return buf[:prefixLen+suffixLen]
+	}
+	return make([]byte, prefixLen+suffixLen)
+}
+
 // TODO(peter): make the value size configurable. See valueSizeDist in
 // config.go.
 func (g *generator) randValue(min, max int) []byte {
+	n := min
+	if max > min {
+		n += g.rng.Intn(max - min)
+	}
+	buf := make([]byte, n)
+	g.fillRand(buf)
+	return buf
+}
+
+func (g *generator) fillRand(buf []byte) {
 	// NB: The actual random values are not particularly important. We only use
 	// lowercase letters because that makes visual determination of ordering
 	// easier, rather than having to remember the lexicographic ordering of
@@ -178,13 +257,6 @@ func (g *generator) randValue(min, max int) []byte {
 	const lettersLen = uint64(len(letters))
 	const lettersCharsPerRand = 12 // floor(log(math.MaxUint64)/log(lettersLen))
 
-	n := min
-	if max > min {
-		n += g.rng.Intn(max - min)
-	}
-
-	buf := make([]byte, n)
-
 	var r uint64
 	var q int
 	for i := 0; i < len(buf); i++ {
@@ -196,8 +268,6 @@ func (g *generator) randValue(min, max int) []byte {
 		r = r / lettersLen
 		q--
 	}
-
-	return buf
 }
 
 func (g *generator) newBatch() {
@@ -288,7 +358,7 @@ func (g *generator) dbCompact() {
 	// Generate new key(s) with a 1% probability.
 	start := g.randKeyToRead(0.01)
 	end := g.randKeyToRead(0.01)
-	if bytes.Compare(start, end) > 0 {
+	if g.cmp(start, end) > 0 {
 		start, end = end, start
 	}
 	g.add(&compactOp{
@@ -346,7 +416,7 @@ func (g *generator) newIter() {
 		// Generate a new key with a .1% probability.
 		upper = g.randKeyToRead(0.001)
 	}
-	if bytes.Compare(lower, upper) > 0 {
+	if g.cmp(lower, upper) > 0 {
 		lower, upper = upper, lower
 	}
 
@@ -434,10 +504,10 @@ func (g *generator) iterSetBounds() {
 			// Generate a new key with a .1% probability.
 			upper = g.randKeyToRead(0.001)
 		}
-		if bytes.Compare(lower, upper) > 0 {
+		if g.cmp(lower, upper) > 0 {
 			lower, upper = upper, lower
 		}
-		if ensureLowerGE && bytes.Compare(iterLastBounds.upper, lower) > 0 {
+		if ensureLowerGE && g.cmp(iterLastBounds.upper, lower) > 0 {
 			if attempts < 25 {
 				continue
 			}
@@ -445,7 +515,7 @@ func (g *generator) iterSetBounds() {
 			upper = lower
 			break
 		}
-		if ensureUpperLE && bytes.Compare(upper, iterLastBounds.lower) > 0 {
+		if ensureUpperLE && g.cmp(upper, iterLastBounds.lower) > 0 {
 			if attempts < 25 {
 				continue
 			}
@@ -540,7 +610,7 @@ func (g *generator) iterSeekGEWithLimit() {
 	}
 	// 0.1% new keys
 	key, limit := g.randKeyToRead(0.001), g.randKeyToRead(0.001)
-	if bytes.Compare(key, limit) > 0 {
+	if g.cmp(key, limit) > 0 {
 		key, limit = limit, key
 	}
 	g.add(&iterSeekGEOp{
@@ -578,7 +648,7 @@ func (g *generator) iterSeekLTWithLimit() {
 	}
 	// 0.1% new keys
 	key, limit := g.randKeyToRead(0.001), g.randKeyToRead(0.001)
-	if bytes.Compare(limit, key) > 0 {
+	if g.cmp(limit, key) > 0 {
 		key, limit = limit, key
 	}
 	g.add(&iterSeekLTOp{
@@ -743,7 +813,7 @@ func (g *generator) writerDeleteRange() {
 
 	start := g.randKeyToWrite(0.001)
 	end := g.randKeyToWrite(0.001)
-	if bytes.Compare(start, end) > 0 {
+	if g.cmp(start, end) > 0 {
 		start, end = end, start
 	}
 
@@ -862,6 +932,10 @@ func (g *generator) tryRepositionBatchIters(writerID objID) {
 	}
 }
 
+func (g *generator) cmp(a, b []byte) int {
+	return g.keyManager.comparer.Compare(a, b)
+}
+
 func (g *generator) String() string {
 	var buf bytes.Buffer
 	for _, op := range g.ops {
diff --git a/internal/metamorphic/key_manager.go b/internal/metamorphic/key_manager.go
index 18e8af4d..371deb61 100644
--- a/internal/metamorphic/key_manager.go
+++ b/internal/metamorphic/key_manager.go
@@ -1,9 +1,11 @@
 package metamorphic
 
 import (
-	"bytes"
 	"fmt"
 	"sort"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/testkeys"
 )
 
 // objKey is a tuple of (objID, key). This struct is used primarily as a map
@@ -96,6 +98,8 @@ func (m *keyMeta) mergeInto(other *keyMeta) {
 // are what we cannot tolerate (doing SingleDelete on a key that has not been
 // written to because the Set was lost is harmless).
 type keyManager struct {
+	comparer *base.Comparer
+
 	// byObjKey tracks the state for each (writer, key) pair. It refers to the
 	// same *keyMeta as in the byObj slices. Using a map allows for fast state
 	// lookups when changing the state based on a writer operation on the key.
@@ -112,6 +116,12 @@ type keyManager struct {
 	// writers, including inflight state that has not made its way to the DB
 	// yet.The keyMeta.objKey is uninitialized.
 	globalKeysMap map[string]*keyMeta
+	// globalKeyPrefixes contains all the key prefixes (as defined by the
+	// comparer's Split) generated so far.
+	globalKeyPrefixes [][]byte
+	// globalKeyPrefixesMap contains the same keys as globalKeyPrefixes. It
+	// ensures no duplication.
+	globalKeyPrefixesMap map[string]struct{}
 
 	// Using SingleDeletes imposes some constraints on the above state, and
 	// causes some state transitions that help with generating complex but
@@ -161,23 +171,32 @@ var dbObjID objID = makeObjID(dbTag, 0)
 // canTolerateApplyFailure methods only.
 func newKeyManager() *keyManager {
 	m := &keyManager{
-		byObjKey:      make(map[string]*keyMeta),
-		byObj:         make(map[objID][]*keyMeta),
-		globalKeysMap: make(map[string]*keyMeta),
+		comparer:             testkeys.Comparer,
+		byObjKey:             make(map[string]*keyMeta),
+		byObj:                make(map[objID][]*keyMeta),
+		globalKeysMap:        make(map[string]*keyMeta),
+		globalKeyPrefixesMap: make(map[string]struct{}),
 	}
 	m.byObj[dbObjID] = []*keyMeta{}
 	return m
 }
 
-// addKey adds the given key to the key manager for global key tracking.
+// addNewKey adds the given key to the key manager for global key tracking.
 // Returns false iff this is not a new key.
 func (k *keyManager) addNewKey(key []byte) bool {
 	_, ok := k.globalKeysMap[string(key)]
 	if ok {
 		return false
 	}
+	keyString := string(key)
 	k.globalKeys = append(k.globalKeys, key)
-	k.globalKeysMap[string(key)] = &keyMeta{objKey: objKey{key: key}}
+	k.globalKeysMap[keyString] = &keyMeta{objKey: objKey{key: key}}
+
+	prefixLen := k.comparer.Split(key)
+	if _, ok := k.globalKeyPrefixesMap[keyString[:prefixLen]]; !ok {
+		k.globalKeyPrefixes = append(k.globalKeyPrefixes, key[:prefixLen])
+		k.globalKeyPrefixesMap[keyString[:prefixLen]] = struct{}{}
+	}
 	return true
 }
 
@@ -357,6 +376,17 @@ func (k *keyManager) eligibleReadKeys() (keys [][]byte) {
 	return k.globalKeys
 }
 
+func (k *keyManager) prefixes() (prefixes [][]byte) {
+	return k.globalKeyPrefixes
+}
+
+// prefixExists returns true if a key has been generated with the provided
+// prefix before.
+func (k *keyManager) prefixExists(prefix []byte) bool {
+	_, exists := k.globalKeyPrefixesMap[string(prefix)]
+	return exists
+}
+
 func (k *keyManager) eligibleWriteKeys() (keys [][]byte) {
 	// Creating and sorting this slice of keys is wasteful given that the
 	// caller will pick one, but makes it simpler for unit testing.
@@ -367,7 +397,7 @@ func (k *keyManager) eligibleWriteKeys() (keys [][]byte) {
 		keys = append(keys, v.key)
 	}
 	sort.Slice(keys, func(i, j int) bool {
-		return bytes.Compare(keys[i], keys[j]) < 0
+		return k.comparer.Compare(keys[i], keys[j]) < 0
 	})
 	return keys
 }
@@ -389,7 +419,7 @@ func (k *keyManager) eligibleSingleDeleteKeys(id objID) (keys [][]byte) {
 		addForObjID(dbObjID)
 	}
 	sort.Slice(keys, func(i, j int) bool {
-		return bytes.Compare(keys[i], keys[j]) < 0
+		return k.comparer.Compare(keys[i], keys[j]) < 0
 	})
 	return keys
 }
diff --git a/internal/metamorphic/options.go b/internal/metamorphic/options.go
index a25af700..ab24b037 100644
--- a/internal/metamorphic/options.go
+++ b/internal/metamorphic/options.go
@@ -9,18 +9,11 @@ import (
 	"github.com/cockroachdb/pebble/bloom"
 	"github.com/cockroachdb/pebble/internal/cache"
 	"github.com/cockroachdb/pebble/internal/randvar"
+	"github.com/cockroachdb/pebble/internal/testkeys"
 	"github.com/cockroachdb/pebble/vfs"
 	"golang.org/x/exp/rand"
 )
 
-var comparer = func() pebble.Comparer {
-	c := *pebble.DefaultComparer
-	c.Split = func(a []byte) int {
-		return len(a)
-	}
-	return c
-}()
-
 func parseOptions(opts *testOptions, data string) error {
 	hooks := &pebble.ParseHooks{
 		NewCache: pebble.NewCache,
@@ -77,7 +70,7 @@ func optionsToString(opts *testOptions) string {
 
 func defaultOptions() *pebble.Options {
 	opts := &pebble.Options{
-		Comparer:           &comparer,
+		Comparer:           testkeys.Comparer,
 		FS:                 vfs.NewMem(),
 		FormatMajorVersion: pebble.FormatNewest,
 		Levels: []pebble.LevelOptions{{


Test patch:
diff --git a/internal/metamorphic/generator_test.go b/internal/metamorphic/generator_test.go
index f39da06d..afae51bd 100644
--- a/internal/metamorphic/generator_test.go
+++ b/internal/metamorphic/generator_test.go
@@ -16,7 +16,7 @@ import (
 
 func TestGenerator(t *testing.T) {
 	rng := randvar.NewRand()
-	g := newGenerator(rng)
+	g := newGenerator(rng, defaultConfig())
 
 	g.newBatch()
 	g.newBatch()
@@ -61,7 +61,7 @@ func TestGenerator(t *testing.T) {
 		t.Logf("\n%s", g)
 	}
 
-	g = newGenerator(rng)
+	g = newGenerator(rng, defaultConfig())
 
 	g.newSnapshot()
 	g.newSnapshot()
@@ -94,7 +94,7 @@ func TestGenerator(t *testing.T) {
 		t.Logf("\n%s", g)
 	}
 
-	g = newGenerator(rng)
+	g = newGenerator(rng, defaultConfig())
 
 	g.newIndexedBatch()
 	g.newIndexedBatch()
@@ -129,7 +129,7 @@ func TestGeneratorRandom(t *testing.T) {
 	generateFromSeed := func() string {
 		rng := rand.New(rand.NewSource(seed))
 		count := ops.Uint64(rng)
-		return formatOps(generate(rng, count, defaultConfig))
+		return formatOps(generate(rng, count, defaultConfig()))
 	}
 
 	// Ensure that generate doesn't use any other source of randomness other
diff --git a/internal/metamorphic/key_manager_test.go b/internal/metamorphic/key_manager_test.go
index eb90c617..9c591995 100644
--- a/internal/metamorphic/key_manager_test.go
+++ b/internal/metamorphic/key_manager_test.go
@@ -169,12 +169,37 @@ func TestKeyManager_AddKey(t *testing.T) {
 	k1 := []byte("foo")
 	require.True(t, m.addNewKey(k1))
 	require.Len(t, m.globalKeys, 1)
+	require.Len(t, m.globalKeyPrefixes, 1)
 	require.Contains(t, m.globalKeys, k1)
+	require.Contains(t, m.globalKeyPrefixes, k1)
 	require.False(t, m.addNewKey(k1))
+	require.True(t, m.prefixExists([]byte("foo")))
+	require.False(t, m.prefixExists([]byte("bar")))
+
 	k2 := []byte("bar")
 	require.True(t, m.addNewKey(k2))
 	require.Len(t, m.globalKeys, 2)
+	require.Len(t, m.globalKeyPrefixes, 2)
 	require.Contains(t, m.globalKeys, k2)
+	require.Contains(t, m.globalKeyPrefixes, k2)
+	require.True(t, m.prefixExists([]byte("bar")))
+	k3 := []byte("bax@4")
+	require.True(t, m.addNewKey(k3))
+	require.Len(t, m.globalKeys, 3)
+	require.Len(t, m.globalKeyPrefixes, 3)
+	require.Contains(t, m.globalKeys, k3)
+	require.Contains(t, m.globalKeyPrefixes, []byte("bax"))
+	require.True(t, m.prefixExists([]byte("bax")))
+	k4 := []byte("foo@6")
+	require.True(t, m.addNewKey(k4))
+	require.Len(t, m.globalKeys, 4)
+	require.Len(t, m.globalKeyPrefixes, 3)
+	require.Contains(t, m.globalKeys, k4)
+	require.True(t, m.prefixExists([]byte("foo")))
+
+	require.Equal(t, [][]byte{
+		[]byte("foo"), []byte("bar"), []byte("bax"),
+	}, m.prefixes())
 }
 
 func TestKeyManager_GetOrInit(t *testing.T) {
diff --git a/internal/metamorphic/meta_test.go b/internal/metamorphic/meta_test.go
index cf2fc7a4..bfc1d213 100644
--- a/internal/metamorphic/meta_test.go
+++ b/internal/metamorphic/meta_test.go
@@ -21,6 +21,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/errorfs"
 	"github.com/cockroachdb/pebble/internal/randvar"
+	"github.com/cockroachdb/pebble/internal/testkeys"
 	"github.com/cockroachdb/pebble/vfs"
 	"github.com/pmezard/go-difflib/difflib"
 	"github.com/stretchr/testify/require"
@@ -135,8 +136,9 @@ func testMetaRun(t *testing.T, runDir string, seed uint64, historyPath string) {
 	testOpts := &testOptions{opts: opts}
 	require.NoError(t, parseOptions(testOpts, string(optionsData)))
 
-	// Always use our custom comparer which provides a Split method.
-	opts.Comparer = &comparer
+	// Always use our custom comparer which provides a Split method, splitting
+	// keys at the trailing '@'.
+	opts.Comparer = testkeys.Comparer
 	// Use an archive cleaner to ease post-mortem debugging.
 	opts.Cleaner = base.ArchiveCleaner{}
 
@@ -263,7 +265,7 @@ func TestMeta(t *testing.T) {
 
 	// Generate a new set of random ops, writing them to <dir>/ops. These will be
 	// read by the child processes when performing a test run.
-	ops := generate(rng, opCount, defaultConfig)
+	ops := generate(rng, opCount, defaultConfig())
 	opsPath := filepath.Join(metaDir, "ops")
 	formattedOps := formatOps(ops)
 	require.NoError(t, ioutil.WriteFile(opsPath, []byte(formattedOps), 0644))
diff --git a/internal/metamorphic/parser_test.go b/internal/metamorphic/parser_test.go
index 7d7de9fc..ff971c6e 100644
--- a/internal/metamorphic/parser_test.go
+++ b/internal/metamorphic/parser_test.go
@@ -31,7 +31,7 @@ func TestParser(t *testing.T) {
 }
 
 func TestParserRandom(t *testing.T) {
-	ops := generate(randvar.NewRand(), 10000, defaultConfig)
+	ops := generate(randvar.NewRand(), 10000, defaultConfig())
 	src := formatOps(ops)
 
 	parsedOps, err := parse([]byte(src))
diff --git a/internal/testkeys/testkeys.go b/internal/testkeys/testkeys.go
index 67b84c35..7ac47abf 100644
--- a/internal/testkeys/testkeys.go
+++ b/internal/testkeys/testkeys.go
@@ -175,6 +175,18 @@ func Suffix(t int) []byte {
 	return b[:WriteSuffix(b, t)]
 }
 
+// SuffixLen returns the exact length of the given suffix when encoded.
+func SuffixLen(t int) int {
+	// Begin at 1 for the '@' delimiter, 1 for a single digit.
+	n := 2
+	t /= 10
+	for t > 0 {
+		t /= 10
+		n++
+	}
+	return n
+}
+
 // WriteSuffix writes the test keys suffix representation of timestamp t to dst,
 // returning the number of bytes written.
 func WriteSuffix(dst []byte, t int) int {
diff --git a/internal/testkeys/testkeys_test.go b/internal/testkeys/testkeys_test.go
index dbf1c6ce..9e8aede0 100644
--- a/internal/testkeys/testkeys_test.go
+++ b/internal/testkeys/testkeys_test.go
@@ -153,6 +153,28 @@ func TestSuffix(t *testing.T) {
 	}
 }
 
+func TestSuffixLen(t *testing.T) {
+	testCases := map[int]int{
+		0:    2,
+		1:    2,
+		5:    2,
+		9:    2,
+		10:   3,
+		17:   3,
+		20:   3,
+		99:   3,
+		100:  4,
+		101:  4,
+		999:  4,
+		1000: 5,
+	}
+	for ts, want := range testCases {
+		if got := SuffixLen(ts); got != want {
+			t.Errorf("SuffixLen(%d) = %d, want %d", ts, got, want)
+		}
+	}
+}
+
 func keyspaceToString(ks Keyspace) string {
 	var buf bytes.Buffer
 	b := make([]byte, ks.MaxLen())