Repository: cockroachdb/pebble
Pull request #1635 (2022-04-08 17:23:21)

internal/metamorphic: add support for an initial database state
**internal/metamorphic: use a wal_dir within the data directory**

When constructing database options to exercise the `wal_dir` option, use a
subdirectory of the `data` directory. This ensures that the `--keep`
command-line flag preserves these runs' WALs. Previously, `--keep` would
erroneously not preserve log files for runs with the `wal_dir` option set.

**vfs: add CloneSync, CloneSkip options for vfs.Clone**

**internal/metamorphic: add support for an initial database state**

Adjust the metamorphic tests to support three new command-line flags:

`--previous-ops` — The path of a previous metamorphic test's `ops` file. The
ops file is read to prepopulate the op generator's key set, ensuring the test
generates operations that frequently draw from the keys that already exist in
the database.

`--initial-state` — The path of a previous metamorphic run's `data` directory,
containing the state at the end of the test. Before each run of the metamorphic
test, the contents of the initial state directory are copied into the test's
VFS.

`--initial-state-desc` — A human-readable string describing the origin of
`initial-state`. This parameter has no functional purpose, but it's persisted
within the `OPTIONS` file to record the historical lineage of a test database
to aid in reproduction.

Subsequent work will build a small harness that takes multiple
`metamorphic.test` binaries (eg, generated from
`go test -c ./internal/metamorphic`) built from separate SHAs,
and invokes the metamorphic tests, propagating data directories from
older SHA runs to newer SHA runs.

Informs #1612.

cockroach-teamcity: 

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.png" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/cockroachdb/pebble/1635)
<!-- Reviewable:end -->


Patch:
diff --git a/internal/metamorphic/generator.go b/internal/metamorphic/generator.go
index 53b0df8b..e43708d0 100644
--- a/internal/metamorphic/generator.go
+++ b/internal/metamorphic/generator.go
@@ -64,12 +64,12 @@ type generator struct {
 	snapshots map[objID]objIDSet
 }
 
-func newGenerator(rng *rand.Rand, cfg config) *generator {
+func newGenerator(rng *rand.Rand, cfg config, km *keyManager) *generator {
 	g := &generator{
 		cfg:             cfg,
 		rng:             rng,
 		init:            &initOp{},
-		keyManager:      newKeyManager(),
+		keyManager:      km,
 		liveReaders:     objIDSlice{makeObjID(dbTag, 0)},
 		liveWriters:     objIDSlice{makeObjID(dbTag, 0)},
 		batches:         make(map[objID]objIDSet),
@@ -83,8 +83,8 @@ func newGenerator(rng *rand.Rand, cfg config) *generator {
 	return g
 }
 
-func generate(rng *rand.Rand, count uint64, cfg config) []op {
-	g := newGenerator(rng, cfg)
+func generate(rng *rand.Rand, count uint64, cfg config, km *keyManager) []op {
+	g := newGenerator(rng, cfg, km)
 
 	generators := []func(){
 		batchAbort:           g.batchAbort,
diff --git a/internal/metamorphic/key_manager.go b/internal/metamorphic/key_manager.go
index 371deb61..6a094d81 100644
--- a/internal/metamorphic/key_manager.go
+++ b/internal/metamorphic/key_manager.go
@@ -3,9 +3,11 @@ package metamorphic
 import (
 	"fmt"
 	"sort"
+	"testing"
 
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/testkeys"
+	"github.com/stretchr/testify/require"
 )
 
 // objKey is a tuple of (objID, key). This struct is used primarily as a map
@@ -446,3 +448,70 @@ func (k *keyManager) canTolerateApplyFailure(id objID) bool {
 	}
 	return true
 }
+
+func opWrittenKeys(untypedOp op) [][]byte {
+	switch t := untypedOp.(type) {
+	case *applyOp:
+	case *batchCommitOp:
+	case *checkpointOp:
+	case *closeOp:
+	case *compactOp:
+	case *dbRestartOp:
+	case *deleteOp:
+		return [][]byte{t.key}
+	case *deleteRangeOp:
+		return [][]byte{t.start, t.end}
+	case *flushOp:
+	case *getOp:
+	case *ingestOp:
+	case *initOp:
+	case *iterFirstOp:
+	case *iterLastOp:
+	case *iterNextOp:
+	case *iterPrevOp:
+	case *iterSeekGEOp:
+	case *iterSeekLTOp:
+	case *iterSeekPrefixGEOp:
+	case *iterSetBoundsOp:
+	case *iterSetOptionsOp:
+	case *mergeOp:
+		return [][]byte{t.key}
+	case *newBatchOp:
+	case *newIndexedBatchOp:
+	case *newIterOp:
+	case *newIterUsingCloneOp:
+	case *newSnapshotOp:
+	case *rangeKeyDeleteOp:
+	case *rangeKeySetOp:
+	case *rangeKeyUnsetOp:
+	case *setOp:
+		return [][]byte{t.key}
+	case *singleDeleteOp:
+		return [][]byte{t.key}
+	}
+	return nil
+}
+
+func loadPrecedingKeys(t testing.TB, ops []op, cfg *config, m *keyManager) {
+	for _, op := range ops {
+		// Pretend we're generating all the operation's keys as potential new
+		// key, so that we update the key manager's keys and prefix sets.
+		for _, k := range opWrittenKeys(op) {
+			m.addNewKey(k)
+
+			// If the key has a suffix, ratchet up the suffix distribution if
+			// necessary.
+			if s := m.comparer.Split(k); s < len(k) {
+				suffix, err := testkeys.ParseSuffix(k[s:])
+				require.NoError(t, err)
+				if uint64(suffix) > cfg.writeSuffixDist.Max() {
+					diff := int(uint64(suffix) - cfg.writeSuffixDist.Max())
+					cfg.writeSuffixDist.IncMax(diff)
+				}
+			}
+		}
+
+		// Update key tracking state.
+		m.update(op)
+	}
+}
diff --git a/internal/metamorphic/options.go b/internal/metamorphic/options.go
index 36a58703..76499478 100644
--- a/internal/metamorphic/options.go
+++ b/internal/metamorphic/options.go
@@ -5,6 +5,11 @@
 package metamorphic
 
 import (
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+
 	"github.com/cockroachdb/pebble"
 	"github.com/cockroachdb/pebble/bloom"
 	"github.com/cockroachdb/pebble/internal/cache"
@@ -22,7 +27,7 @@ func parseOptions(opts *testOptions, data string) error {
 			}
 			return bloom.FilterPolicy(10), nil
 		},
-		SkipUnknown: func(name string) bool {
+		SkipUnknown: func(name, value string) bool {
 			switch name {
 			case "TestOptions":
 				return true
@@ -38,6 +43,12 @@ func parseOptions(opts *testOptions, data string) error {
 			case "TestOptions.use_disk":
 				opts.useDisk = true
 				return true
+			case "TestOptions.initial_state_desc":
+				opts.initialStateDesc = value
+				return true
+			case "TestOptions.initial_state_path":
+				opts.initialStatePath = value
+				return true
 			default:
 				return false
 			}
@@ -48,23 +59,31 @@ func parseOptions(opts *testOptions, data string) error {
 }
 
 func optionsToString(opts *testOptions) string {
-	str := opts.opts.String()
-	if opts.strictFS || opts.ingestUsingApply || opts.replaceSingleDelete || opts.useDisk {
-		str += "\n[TestOptions]\n"
-	}
+	var buf bytes.Buffer
 	if opts.strictFS {
-		str += "  strictfs=true\n"
+		fmt.Fprint(&buf, "  strictfs=true\n")
 	}
 	if opts.ingestUsingApply {
-		str += "  ingest_using_apply=true\n"
+		fmt.Fprint(&buf, "  ingest_using_apply=true\n")
 	}
 	if opts.replaceSingleDelete {
-		str += "  replace_single_delete=true\n"
+		fmt.Fprint(&buf, "  replace_single_delete=true\n")
 	}
 	if opts.useDisk {
-		str += "  use_disk=true\n"
+		fmt.Fprint(&buf, "  use_disk=true\n")
+	}
+	if opts.initialStatePath != "" {
+		fmt.Fprintf(&buf, "  initial_state_path=%s\n", opts.initialStatePath)
+	}
+	if opts.initialStateDesc != "" {
+		fmt.Fprintf(&buf, "  initial_state_desc=%s\n", opts.initialStateDesc)
 	}
-	return str
+
+	s := opts.opts.String()
+	if buf.Len() == 0 {
+		return s
+	}
+	return s + "\n[TestOptions]\n" + buf.String()
 }
 
 func defaultOptions() *pebble.Options {
@@ -88,6 +107,12 @@ type testOptions struct {
 	ingestUsingApply bool
 	// Replace a SINGLEDEL with a DELETE.
 	replaceSingleDelete bool
+	// The path on the local filesystem where the initial state of the database
+	// exists.  Empty if the test run begins from an empty database state.
+	initialStatePath string
+	// A human-readable string describing the initial state of the database.
+	// Empty if the test run begins from an empty database state.
+	initialStateDesc string
 }
 
 func standardOptions() []*testOptions {
@@ -134,7 +159,7 @@ func standardOptions() []*testOptions {
 `,
 		10: `
 [Options]
-  wal_dir=wal
+  wal_dir=data/wal
 `,
 		11: `
 [Level "0"]
@@ -224,7 +249,7 @@ func randomOptions(rng *rand.Rand) *testOptions {
 	opts.MemTableSize = 2 << (10 + uint(rng.Intn(16))) // 2KB - 256MB
 	opts.MemTableStopWritesThreshold = 2 + rng.Intn(5) // 2 - 5
 	if rng.Intn(2) == 0 {
-		opts.WALDir = "wal"
+		opts.WALDir = "data/wal"
 	}
 	var lopts pebble.LevelOptions
 	lopts.BlockRestartInterval = 1 + rng.Intn(64)  // 1 - 64
@@ -247,3 +272,59 @@ func randomOptions(rng *rand.Rand) *testOptions {
 	testOpts.replaceSingleDelete = rng.Intn(2) != 0
 	return testOpts
 }
+
+func setupInitialState(dir string, testOpts *testOptions) error {
+	// Copy (vfs.Default,<initialStatePath>) to (testOpts.opts.FS,<dir>).
+	ok, err := vfs.Clone(
+		vfs.Default,
+		testOpts.opts.FS,
+		testOpts.initialStatePath,
+		dir,
+		vfs.CloneSync,
+		vfs.CloneSkip(func(filename string) bool {
+			// Skip the archive of historical files, any checkpoints created by
+			// operations and files staged for ingest in tmp.
+			b := filepath.Base(filename)
+			return b == "archive" || b == "checkpoints" || b == "tmp"
+		}))
+	if err != nil {
+		return err
+	} else if !ok {
+		return os.ErrNotExist
+	}
+
+	// Tests with wal_dir set store their WALs in a `wal` directory. The source
+	// database (initialStatePath) could've had wal_dir set, or the current test
+	// options (testOpts) could have wal_dir set, or both.
+	fs := testOpts.opts.FS
+	walDir := fs.PathJoin(dir, "wal")
+	if err := fs.MkdirAll(walDir, os.ModePerm); err != nil {
+		return err
+	}
+
+	// Copy <dir>/wal/*.log -> <dir>.
+	src, dst := walDir, dir
+	if testOpts.opts.WALDir != "" {
+		// Copy <dir>/*.log -> <dir>/wal.
+		src, dst = dst, src
+	}
+	return moveLogs(fs, src, dst)
+}
+
+func moveLogs(fs vfs.FS, srcDir, dstDir string) error {
+	ls, err := fs.List(srcDir)
+	if err != nil {
+		return err
+	}
+	for _, f := range ls {
+		if filepath.Ext(f) != ".log" {
+			continue
+		}
+		src := fs.PathJoin(srcDir, f)
+		dst := fs.PathJoin(dstDir, f)
+		if err := fs.Rename(src, dst); err != nil {
+			return err
+		}
+	}
+	return nil
+}
diff --git a/options.go b/options.go
index 4f1189f7..d9f43e29 100644
--- a/options.go
+++ b/options.go
@@ -1009,7 +1009,7 @@ type ParseHooks struct {
 	NewComparer     func(name string) (*Comparer, error)
 	NewFilterPolicy func(name string) (FilterPolicy, error)
 	NewMerger       func(name string) (*Merger, error)
-	SkipUnknown     func(name string) bool
+	SkipUnknown     func(name, value string) bool
 }
 
 // Parse parses the options from the specified string. Note that certain
@@ -1027,7 +1027,7 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
 			switch key {
 			case "pebble_version":
 			default:
-				if hooks != nil && hooks.SkipUnknown != nil && hooks.SkipUnknown(section+"."+key) {
+				if hooks != nil && hooks.SkipUnknown != nil && hooks.SkipUnknown(section+"."+key, value) {
 					return nil
 				}
 				return errors.Errorf("pebble: unknown option: %s.%s",
@@ -1155,7 +1155,7 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
 			case "wal_bytes_per_sync":
 				o.WALBytesPerSync, err = strconv.Atoi(value)
 			default:
-				if hooks != nil && hooks.SkipUnknown != nil && hooks.SkipUnknown(section+"."+key) {
+				if hooks != nil && hooks.SkipUnknown != nil && hooks.SkipUnknown(section+"."+key, value) {
 					return nil
 				}
 				return errors.Errorf("pebble: unknown option: %s.%s",
@@ -1168,7 +1168,7 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
 			if n, err := fmt.Sscanf(section, `Level "%d"`, &index); err != nil {
 				return err
 			} else if n != 1 {
-				if hooks != nil && hooks.SkipUnknown != nil && hooks.SkipUnknown(section) {
+				if hooks != nil && hooks.SkipUnknown != nil && hooks.SkipUnknown(section, value) {
 					return nil
 				}
 				return errors.Errorf("pebble: unknown section: %q", errors.Safe(section))
@@ -1216,14 +1216,14 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
 			case "target_file_size":
 				l.TargetFileSize, err = strconv.ParseInt(value, 10, 64)
 			default:
-				if hooks != nil && hooks.SkipUnknown != nil && hooks.SkipUnknown(section+"."+key) {
+				if hooks != nil && hooks.SkipUnknown != nil && hooks.SkipUnknown(section+"."+key, value) {
 					return nil
 				}
 				return errors.Errorf("pebble: unknown option: %s.%s", errors.Safe(section), errors.Safe(key))
 			}
 			return err
 		}
-		if hooks != nil && hooks.SkipUnknown != nil && hooks.SkipUnknown(section+"."+key) {
+		if hooks != nil && hooks.SkipUnknown != nil && hooks.SkipUnknown(section+"."+key, value) {
 			return nil
 		}
 		return errors.Errorf("pebble: unknown section: %q", errors.Safe(section))
diff --git a/tool/db.go b/tool/db.go
index e843b41a..8295979f 100644
--- a/tool/db.go
+++ b/tool/db.go
@@ -200,7 +200,7 @@ func (d *dbT) loadOptions(dir string) error {
 			}
 			return nil, errors.Errorf("unknown merger %q", errors.Safe(name))
 		},
-		SkipUnknown: func(name string) bool {
+		SkipUnknown: func(name, value string) bool {
 			return true
 		},
 	}
diff --git a/vfs/clone.go b/vfs/clone.go
index 8604b714..77a5992a 100644
--- a/vfs/clone.go
+++ b/vfs/clone.go
@@ -11,13 +11,35 @@ import (
 	"github.com/cockroachdb/errors/oserror"
 )
 
+type cloneOpts struct {
+	skip func(string) bool
+	sync bool
+}
+
+// A CloneOption configures the behavior of Clone.
+type CloneOption func(*cloneOpts)
+
+// CloneSkip configures Clone to skip files for which the provided function
+// returns true when passed the file's path.
+func CloneSkip(fn func(string) bool) CloneOption {
+	return func(co *cloneOpts) { co.skip = fn }
+}
+
+// CloneSync configures Clone to sync files and directories.
+var CloneSync CloneOption = func(o *cloneOpts) { o.sync = true }
+
 // Clone recursively copies a directory structure from srcFS to dstFS. srcPath
 // specifies the path in srcFS to copy from and must be compatible with the
 // srcFS path format. dstDir is the target directory in dstFS and must be
 // compatible with the dstFS path format. Returns (true,nil) on a successful
 // copy, (false,nil) if srcPath does not exist, and (false,err) if an error
 // occurred.
-func Clone(srcFS, dstFS FS, srcPath, dstPath string) (bool, error) {
+func Clone(srcFS, dstFS FS, srcPath, dstPath string, opts ...CloneOption) (bool, error) {
+	var o cloneOpts
+	for _, opt := range opts {
+		opt(&o)
+	}
+
 	srcFile, err := srcFS.Open(srcPath)
 	if err != nil {
 		if oserror.IsNotExist(err) {
@@ -47,11 +69,28 @@ func Clone(srcFS, dstFS FS, srcPath, dstPath string) (bool, error) {
 		// Sort the paths so we get deterministic test output.
 		sort.Strings(list)
 		for _, name := range list {
-			_, err := Clone(srcFS, dstFS, srcFS.PathJoin(srcPath, name), dstFS.PathJoin(dstPath, name))
+			if o.skip != nil && o.skip(srcFS.PathJoin(srcPath, name)) {
+				continue
+			}
+			_, err := Clone(srcFS, dstFS, srcFS.PathJoin(srcPath, name), dstFS.PathJoin(dstPath, name), opts...)
 			if err != nil {
 				return false, err
 			}
 		}
+
+		if o.sync {
+			dir, err := dstFS.OpenDir(dstPath)
+			if err != nil {
+				return false, err
+			}
+			if err := dir.Sync(); err != nil {
+				return false, err
+			}
+			if err := dir.Close(); err != nil {
+				return false, err
+			}
+		}
+
 		return true, nil
 	}
 
@@ -69,6 +108,12 @@ func Clone(srcFS, dstFS FS, srcPath, dstPath string) (bool, error) {
 	if _, err = dstFile.Write(data); err != nil {
 		return false, err
 	}
+	if o.sync {
+		if err := dstFile.Sync(); err != nil {
+			return false, err
+		}
+	}
+
 	if err := dstFile.Close(); err != nil {
 		return false, err
 	}


Test patch:
diff --git a/internal/metamorphic/generator_test.go b/internal/metamorphic/generator_test.go
index afae51bd..9f708649 100644
--- a/internal/metamorphic/generator_test.go
+++ b/internal/metamorphic/generator_test.go
@@ -16,7 +16,7 @@ import (
 
 func TestGenerator(t *testing.T) {
 	rng := randvar.NewRand()
-	g := newGenerator(rng, defaultConfig())
+	g := newGenerator(rng, defaultConfig(), newKeyManager())
 
 	g.newBatch()
 	g.newBatch()
@@ -61,7 +61,7 @@ func TestGenerator(t *testing.T) {
 		t.Logf("\n%s", g)
 	}
 
-	g = newGenerator(rng, defaultConfig())
+	g = newGenerator(rng, defaultConfig(), newKeyManager())
 
 	g.newSnapshot()
 	g.newSnapshot()
@@ -94,7 +94,7 @@ func TestGenerator(t *testing.T) {
 		t.Logf("\n%s", g)
 	}
 
-	g = newGenerator(rng, defaultConfig())
+	g = newGenerator(rng, defaultConfig(), newKeyManager())
 
 	g.newIndexedBatch()
 	g.newIndexedBatch()
@@ -129,7 +129,7 @@ func TestGeneratorRandom(t *testing.T) {
 	generateFromSeed := func() string {
 		rng := rand.New(rand.NewSource(seed))
 		count := ops.Uint64(rng)
-		return formatOps(generate(rng, count, defaultConfig()))
+		return formatOps(generate(rng, count, defaultConfig(), newKeyManager()))
 	}
 
 	// Ensure that generate doesn't use any other source of randomness other
diff --git a/internal/metamorphic/key_manager_test.go b/internal/metamorphic/key_manager_test.go
index 9c591995..bace751d 100644
--- a/internal/metamorphic/key_manager_test.go
+++ b/internal/metamorphic/key_manager_test.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"testing"
 
+	"github.com/cockroachdb/pebble/internal/randvar"
 	"github.com/stretchr/testify/require"
 )
 
@@ -485,3 +486,33 @@ func TestKeyManager(t *testing.T) {
 		})
 	}
 }
+
+func TestOpWrittenKeys(t *testing.T) {
+	for name, info := range methods {
+		t.Run(name, func(t *testing.T) {
+			// Any operations that exist in methods but are not handled in
+			// opWrittenKeys will result in a panic, failing the subtest.
+			opWrittenKeys(info.constructor())
+		})
+	}
+}
+
+func TestLoadPrecedingKeys(t *testing.T) {
+	rng := randvar.NewRand()
+	cfg := defaultConfig()
+	km := newKeyManager()
+	ops := generate(rng, 1000, cfg, km)
+
+	cfg2 := defaultConfig()
+	km2 := newKeyManager()
+	loadPrecedingKeys(t, ops, &cfg2, km2)
+
+	// NB: We can't assert equality, because the original run may not have
+	// ever used the max of the distribution.
+	require.Greater(t, cfg2.writeSuffixDist.Max(), uint64(1))
+
+	// NB: We can't assert equality, because the original run may have generated
+	// keys that it didn't end up using in operations.
+	require.Subset(t, km.globalKeys, km2.globalKeys)
+	require.Subset(t, km.globalKeyPrefixes, km2.globalKeyPrefixes)
+}
diff --git a/internal/metamorphic/meta_test.go b/internal/metamorphic/meta_test.go
index c9a699b8..57a16c9a 100644
--- a/internal/metamorphic/meta_test.go
+++ b/internal/metamorphic/meta_test.go
@@ -69,6 +69,22 @@ var (
 the result of the run from the first options file in the list. Example, -compare
 random-003,standard-000. The dir flag should have the directory containing these directories.
 Example, -dir _meta/200610-203012.077`)
+
+	// The following options may be used for split-version metamorphic testing.
+	// To perform split-version testing, the client runs the metamorphic tests
+	// on an earlier Pebble SHA passing the `--keep` flag. The client then
+	// switches to the later Pebble SHA, setting the below options to point to
+	// the `ops` file and one of the previous run's data directories.
+	previousOps = flag.String("previous-ops", "",
+		"path to an ops file, used to prepopulate the set of keys operations draw from")
+	initialStatePath = flag.String("initial-state", "",
+		"path to a database's data directory, used to prepopulate the test run's databases")
+	initialStateDesc = flag.String("initial-state-desc", "",
+		`a human-readable description of the initial database state.
+		If set this parameter is written to the OPTIONS to aid in
+		debugging. It's intended to describe the lineage of a
+		database's state, including sufficient information for
+		reproduction (eg, SHA, prng seed, etc).`)
 )
 
 func init() {
@@ -159,6 +175,16 @@ func testMetaRun(t *testing.T, runDir string, seed uint64, historyPath string) {
 			opts.FS = vfs.NewMem()
 		}
 	}
+
+	dir := opts.FS.PathJoin(runDir, "data")
+	// Set up the initial database state if configured to start from a non-empty
+	// database. By default tests start from an empty database, but split
+	// version testing may configure a previous metamorphic tests's database
+	// state as the initial state.
+	if testOpts.initialStatePath != "" {
+		require.NoError(t, setupInitialState(dir, testOpts))
+	}
+
 	// Wrap the filesystem with one that will inject errors into read
 	// operations with *errorRate probability.
 	opts.FS = errorfs.Wrap(opts.FS, errorfs.WithProbability(errorfs.OpKindRead, *errorRate))
@@ -170,15 +196,15 @@ func testMetaRun(t *testing.T, runDir string, seed uint64, historyPath string) {
 	historyFile, err := os.Create(historyPath)
 	require.NoError(t, err)
 	defer historyFile.Close()
-
 	writers := []io.Writer{historyFile}
+
 	if testing.Verbose() {
 		writers = append(writers, os.Stdout)
 	}
 	h := newHistory(*failRE, writers...)
 
 	m := newTest(ops)
-	require.NoError(t, m.init(h, opts.FS.PathJoin(runDir, "data"), testOpts))
+	require.NoError(t, m.init(h, dir, testOpts))
 	for m.step(h) {
 		if err := h.Error(); err != nil {
 			fmt.Fprintf(os.Stderr, "Seed: %d\n", seed)
@@ -269,7 +295,21 @@ func TestMeta(t *testing.T) {
 
 	// Generate a new set of random ops, writing them to <dir>/ops. These will be
 	// read by the child processes when performing a test run.
-	ops := generate(rng, opCount, defaultConfig())
+	km := newKeyManager()
+	cfg := defaultConfig()
+	if *previousOps != "" {
+		// During split-version testing, we load keys from an `ops` file
+		// produced by a metamorphic test run of an earlier Pebble version.
+		// Seeding the keys ensure we generate interesting operations, including
+		// ones with key shadowing, merging, etc.
+		opsPath := filepath.Join(filepath.Dir(filepath.Clean(*previousOps)), "ops")
+		opsData, err := ioutil.ReadFile(opsPath)
+		require.NoError(t, err)
+		ops, err := parse(opsData)
+		require.NoError(t, err)
+		loadPrecedingKeys(t, ops, &cfg, km)
+	}
+	ops := generate(rng, opCount, cfg, km)
 	opsPath := filepath.Join(metaDir, "ops")
 	formattedOps := formatOps(ops)
 	require.NoError(t, ioutil.WriteFile(opsPath, []byte(formattedOps), 0644))
@@ -329,28 +369,40 @@ func TestMeta(t *testing.T) {
 		}
 	}
 
-	// Perform runs with the standard options.
+	// Create the standard options.
 	var names []string
 	options := map[string]*testOptions{}
 	for i, opts := range standardOptions() {
 		name := fmt.Sprintf("standard-%03d", i)
 		names = append(names, name)
 		options[name] = opts
-		t.Run(name, func(t *testing.T) {
-			runOptions(t, opts)
-		})
 	}
 
-	// Perform runs with random options. We make an arbitrary choice to run with
-	// as many random options as we have standard options.
+	// Create random options. We make an arbitrary choice to run with as many
+	// random options as we have standard options.
 	nOpts := len(options)
 	for i := 0; i < nOpts; i++ {
 		name := fmt.Sprintf("random-%03d", i)
 		names = append(names, name)
 		opts := randomOptions(rng)
 		options[name] = opts
+	}
+
+	// If the user provided the path to an initial database state to use, update
+	// all the options to pull from it.
+	if *initialStatePath != "" {
+		for _, o := range options {
+			var err error
+			o.initialStatePath, err = filepath.Abs(*initialStatePath)
+			require.NoError(t, err)
+			o.initialStateDesc = *initialStateDesc
+		}
+	}
+
+	// Run the options.
+	for _, name := range names {
 		t.Run(name, func(t *testing.T) {
-			runOptions(t, opts)
+			runOptions(t, options[name])
 		})
 	}
 
diff --git a/internal/metamorphic/options_test.go b/internal/metamorphic/options_test.go
new file mode 100644
index 00000000..c4448b1e
--- /dev/null
+++ b/internal/metamorphic/options_test.go
@@ -0,0 +1,51 @@
+// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package metamorphic
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/cockroachdb/pebble"
+	"github.com/cockroachdb/pebble/internal/testkeys"
+	"github.com/cockroachdb/pebble/vfs"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSetupInitialState(t *testing.T) {
+	// Construct a small database in the test's TempDir.
+	initialStatePath := t.TempDir()
+	{
+		d, err := pebble.Open(initialStatePath, &pebble.Options{})
+		require.NoError(t, err)
+		const maxKeyLen = 2
+		ks := testkeys.Alpha(maxKeyLen)
+		var key [maxKeyLen]byte
+		for i := 0; i < ks.Count(); i++ {
+			n := testkeys.WriteKey(key[:], ks, i)
+			require.NoError(t, d.Set(key[:n], key[:n], pebble.NoSync))
+			if i%100 == 0 {
+				require.NoError(t, d.Flush())
+			}
+		}
+		require.NoError(t, d.Close())
+	}
+	require.NoError(t, vfs.Default.MkdirAll(filepath.Join(initialStatePath, "wal"), os.ModePerm))
+	ls, err := vfs.Default.List(initialStatePath)
+	require.NoError(t, err)
+
+	// setupInitialState with an initial state path set to the test's TempDir
+	// should populate opts.opts.FS with the directory's contents.
+	opts := &testOptions{
+		opts:             defaultOptions(),
+		initialStatePath: initialStatePath,
+		initialStateDesc: "test",
+	}
+	require.NoError(t, setupInitialState("", opts))
+	copied, err := opts.opts.FS.List("")
+	require.NoError(t, err)
+	require.ElementsMatch(t, ls, copied)
+}
diff --git a/internal/metamorphic/parser_test.go b/internal/metamorphic/parser_test.go
index ff971c6e..9ff4d6fe 100644
--- a/internal/metamorphic/parser_test.go
+++ b/internal/metamorphic/parser_test.go
@@ -31,7 +31,7 @@ func TestParser(t *testing.T) {
 }
 
 func TestParserRandom(t *testing.T) {
-	ops := generate(randvar.NewRand(), 10000, defaultConfig())
+	ops := generate(randvar.NewRand(), 10000, defaultConfig(), newKeyManager())
 	src := formatOps(ops)
 
 	parsedOps, err := parse([]byte(src))
diff --git a/internal/testkeys/testkeys.go b/internal/testkeys/testkeys.go
index 7ac47abf..f62afb96 100644
--- a/internal/testkeys/testkeys.go
+++ b/internal/testkeys/testkeys.go
@@ -17,6 +17,7 @@ import (
 	"fmt"
 	"math"
 	"strconv"
+	"strings"
 
 	"github.com/cockroachdb/pebble/internal/base"
 )
@@ -187,6 +188,11 @@ func SuffixLen(t int) int {
 	return n
 }
 
+// ParseSuffix returns the integer representation of the encoded suffix.
+func ParseSuffix(s []byte) (int, error) {
+	return strconv.Atoi(strings.TrimPrefix(string(s), string(suffixDelim)))
+}
+
 // WriteSuffix writes the test keys suffix representation of timestamp t to dst,
 // returning the number of bytes written.
 func WriteSuffix(dst []byte, t int) int {