Repository: cockroachdb/pebble
Pull request #1216 (2021-08-09 19:16:04)

pebble: Preserve one previous manifest
We want to be able to preserve some older manifests for debugging purposes.
By default we use 128MB manifests, so this shouldn't cause too much clutter.

Resolves: https://github.com/cockroachdb/pebble/issues/1184

cockroach-teamcity: 

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.png" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/cockroachdb/pebble/1216)
<!-- Reviewable:end -->


Patch:
diff --git a/Makefile b/Makefile
index 2a2f92ff..1d3bb647 100644
--- a/Makefile
+++ b/Makefile
@@ -45,7 +45,7 @@ generate:
 # temporarily hiding those files.
 mod-update:
 	mkdir -p cmd/pebble/_bak
-	mv cmd/pebble/badger.go cmd/pebble/_bak
+	mv cmd/pebble/{badger}.go cmd/pebble/_bak
 	${GO} get -u
 	${GO} mod tidy
 	${GO} mod vendor
diff --git a/compaction.go b/compaction.go
index 277d59a5..03f8fc32 100644
--- a/compaction.go
+++ b/compaction.go
@@ -2683,8 +2683,22 @@ func (d *DB) doDeleteObsoleteFiles(jobID int) {
 	}
 	d.mu.versions.obsoleteTables = nil
 
-	obsoleteManifests := d.mu.versions.obsoleteManifests
-	d.mu.versions.obsoleteManifests = nil
+	// Sort the manifests cause we want to delete some contiguous prefix
+	// of the older manifests.
+	sort.Slice(d.mu.versions.obsoleteManifests, func(i, j int) bool {
+		return d.mu.versions.obsoleteManifests[i].fileNum <
+			d.mu.versions.obsoleteManifests[j].fileNum
+	})
+
+	var obsoleteManifests []fileInfo
+	manifestsToDelete := len(d.mu.versions.obsoleteManifests) - d.opts.NumPrevManifest
+	if manifestsToDelete > 0 {
+		obsoleteManifests = d.mu.versions.obsoleteManifests[:manifestsToDelete]
+		d.mu.versions.obsoleteManifests = d.mu.versions.obsoleteManifests[manifestsToDelete:]
+		if len(d.mu.versions.obsoleteManifests) == 0 {
+			d.mu.versions.obsoleteManifests = nil
+		}
+	}
 
 	obsoleteOptions := d.mu.versions.obsoleteOptions
 	d.mu.versions.obsoleteOptions = nil
diff --git a/internal/errorfs/errorfs.go b/internal/errorfs/errorfs.go
index efb67153..87693eb5 100644
--- a/internal/errorfs/errorfs.go
+++ b/internal/errorfs/errorfs.go
@@ -5,7 +5,6 @@
 package errorfs
 
 import (
-	"fmt"
 	"io"
 	"math/rand"
 	"os"
@@ -21,73 +20,14 @@ import (
 // ErrInjected is an error artifically injected for testing fs error paths.
 var ErrInjected = errors.New("injected error")
 
-// Op is an enum describing the type of operation.
+// Op is an enum describing the type of operation performed.
 type Op int
 
 const (
-	// OpCreate describes a create file operation.
-	OpCreate Op = iota
-	// OpLink describes a hardlink operation.
-	OpLink
-	// OpOpen describes a file open operation.
-	OpOpen
-	// OpOpenDir describes a directory open operation.
-	OpOpenDir
-	// OpRemove describes a remove file operation.
-	OpRemove
-	// OpRemoveAll describes a recursive remove operation.
-	OpRemoveAll
-	// OpRename describes a rename operation.
-	OpRename
-	// OpReuseForRewrite describes a reuse for rewriting operation.
-	OpReuseForRewrite
-	// OpMkdirAll describes a make directory including parents operation.
-	OpMkdirAll
-	// OpLock describes a lock file operation.
-	OpLock
-	// OpList describes a list directory operation.
-	OpList
-	// OpStat describes a path-based stat operation.
-	OpStat
-	// OpGetDiskUsage describes a disk usage operation.
-	OpGetDiskUsage
-	// OpFileClose describes a close file operation.
-	OpFileClose
-	// OpFileRead describes a file read operation.
-	OpFileRead
-	// OpFileReadAt describes a file seek read operation.
-	OpFileReadAt
-	// OpFileWrite describes a file write operation.
-	OpFileWrite
-	// OpFileStat describes a file stat operation.
-	OpFileStat
-	// OpFileSync describes a file sync operation.
-	OpFileSync
-	// OpFileFlush describes a file flush operation.
-	OpFileFlush
-)
-
-// OpKind returns the operation's kind.
-func (o Op) OpKind() OpKind {
-	switch o {
-	case OpOpen, OpOpenDir, OpList, OpStat, OpGetDiskUsage, OpFileRead, OpFileReadAt, OpFileStat:
-		return OpKindRead
-	case OpCreate, OpLink, OpRemove, OpRemoveAll, OpRename, OpReuseForRewrite, OpMkdirAll, OpLock, OpFileClose, OpFileWrite, OpFileSync, OpFileFlush:
-		return OpKindWrite
-	default:
-		panic(fmt.Sprintf("unrecognized op %v\n", o))
-	}
-}
-
-// OpKind is an enum describing whether an operation is a read or write
-// operation.
-type OpKind int
-
-const (
-	// OpKindRead describes read operations.
-	OpKindRead OpKind = iota
-	// OpKindWrite describes write operations.
-	OpKindWrite
+	// OpRead describes read operations.
+	OpRead Op = iota
+	// OpWrite describes write operations.
+	OpWrite
 )
 
 // OnIndex constructs an injector that returns an error on
@@ -109,7 +49,7 @@ func (ii *InjectIndex) Index() int32 { return atomic.LoadInt32(&ii.index) }
 func (ii *InjectIndex) SetIndex(v int32) { atomic.StoreInt32(&ii.index, v) }
 
 // MaybeError implements the Injector interface.
-func (ii *InjectIndex) MaybeError(_ Op, _ string) error {
+func (ii *InjectIndex) MaybeError(op Op) error {
 	if atomic.AddInt32(&ii.index, -1) == -1 {
 		return errors.WithStack(ErrInjected)
 	}
@@ -120,13 +60,13 @@ func (ii *InjectIndex) MaybeError(_ Op, _ string) error {
 // probability when passed op. It may be passed to Wrap to inject an error
 // into an ErrFS with the provided probability. p should be within the range
 // [0.0,1.0].
-func WithProbability(op OpKind, p float64) Injector {
+func WithProbability(op Op, p float64) Injector {
 	mu := new(sync.Mutex)
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
-	return InjectorFunc(func(currOp Op, _ string) error {
+	return InjectorFunc(func(currOp Op) error {
 		mu.Lock()
 		defer mu.Unlock()
-		if currOp.OpKind() == op && rnd.Float64() < p {
+		if currOp == op && rnd.Float64() < p {
 			return errors.WithStack(ErrInjected)
 		}
 		return nil
@@ -135,18 +75,14 @@ func WithProbability(op OpKind, p float64) Injector {
 
 // InjectorFunc implements the Injector interface for a function with
 // MaybeError's signature.
-type InjectorFunc func(Op, string) error
+type InjectorFunc func(Op) error
 
 // MaybeError implements the Injector interface.
-func (f InjectorFunc) MaybeError(op Op, path string) error { return f(op, path) }
+func (f InjectorFunc) MaybeError(op Op) error { return f(op) }
 
 // Injector injects errors into FS operations.
 type Injector interface {
-	// MaybeError is invoked by an errorfs before an operation is executed. It
-	// is passed an enum indicating the type of operation and a path of the
-	// subject file or directory. If the operation takes two paths (eg,
-	// Rename, Link), the original source path is provided.
-	MaybeError(op Op, path string) error
+	MaybeError(Op) error
 }
 
 // FS implements vfs.FS, injecting errors into
@@ -184,19 +120,19 @@ func (fs *FS) Unwrap() vfs.FS {
 
 // Create implements FS.Create.
 func (fs *FS) Create(name string) (vfs.File, error) {
-	if err := fs.inj.MaybeError(OpCreate, name); err != nil {
+	if err := fs.inj.MaybeError(OpWrite); err != nil {
 		return nil, err
 	}
 	f, err := fs.fs.Create(name)
 	if err != nil {
 		return nil, err
 	}
-	return &errorFile{name, f, fs.inj}, nil
+	return &errorFile{f, fs.inj}, nil
 }
 
 // Link implements FS.Link.
 func (fs *FS) Link(oldname, newname string) error {
-	if err := fs.inj.MaybeError(OpLink, oldname); err != nil {
+	if err := fs.inj.MaybeError(OpWrite); err != nil {
 		return err
 	}
 	return fs.fs.Link(oldname, newname)
@@ -204,14 +140,14 @@ func (fs *FS) Link(oldname, newname string) error {
 
 // Open implements FS.Open.
 func (fs *FS) Open(name string, opts ...vfs.OpenOption) (vfs.File, error) {
-	if err := fs.inj.MaybeError(OpOpen, name); err != nil {
+	if err := fs.inj.MaybeError(OpRead); err != nil {
 		return nil, err
 	}
 	f, err := fs.fs.Open(name)
 	if err != nil {
 		return nil, err
 	}
-	ef := &errorFile{name, f, fs.inj}
+	ef := &errorFile{f, fs.inj}
 	for _, opt := range opts {
 		opt.Apply(ef)
 	}
@@ -220,19 +156,19 @@ func (fs *FS) Open(name string, opts ...vfs.OpenOption) (vfs.File, error) {
 
 // OpenDir implements FS.OpenDir.
 func (fs *FS) OpenDir(name string) (vfs.File, error) {
-	if err := fs.inj.MaybeError(OpOpenDir, name); err != nil {
+	if err := fs.inj.MaybeError(OpRead); err != nil {
 		return nil, err
 	}
 	f, err := fs.fs.OpenDir(name)
 	if err != nil {
 		return nil, err
 	}
-	return &errorFile{name, f, fs.inj}, nil
+	return &errorFile{f, fs.inj}, nil
 }
 
 // GetDiskUsage implements FS.GetDiskUsage.
 func (fs *FS) GetDiskUsage(path string) (vfs.DiskUsage, error) {
-	if err := fs.inj.MaybeError(OpGetDiskUsage, path); err != nil {
+	if err := fs.inj.MaybeError(OpRead); err != nil {
 		return vfs.DiskUsage{}, err
 	}
 	return fs.fs.GetDiskUsage(path)
@@ -259,7 +195,7 @@ func (fs *FS) Remove(name string) error {
 		return nil
 	}
 
-	if err := fs.inj.MaybeError(OpRemove, name); err != nil {
+	if err := fs.inj.MaybeError(OpWrite); err != nil {
 		return err
 	}
 	return fs.fs.Remove(name)
@@ -267,7 +203,7 @@ func (fs *FS) Remove(name string) error {
 
 // RemoveAll implements FS.RemoveAll.
 func (fs *FS) RemoveAll(fullname string) error {
-	if err := fs.inj.MaybeError(OpRemoveAll, fullname); err != nil {
+	if err := fs.inj.MaybeError(OpWrite); err != nil {
 		return err
 	}
 	return fs.fs.RemoveAll(fullname)
@@ -275,7 +211,7 @@ func (fs *FS) RemoveAll(fullname string) error {
 
 // Rename implements FS.Rename.
 func (fs *FS) Rename(oldname, newname string) error {
-	if err := fs.inj.MaybeError(OpRename, oldname); err != nil {
+	if err := fs.inj.MaybeError(OpWrite); err != nil {
 		return err
 	}
 	return fs.fs.Rename(oldname, newname)
@@ -283,7 +219,7 @@ func (fs *FS) Rename(oldname, newname string) error {
 
 // ReuseForWrite implements FS.ReuseForWrite.
 func (fs *FS) ReuseForWrite(oldname, newname string) (vfs.File, error) {
-	if err := fs.inj.MaybeError(OpReuseForRewrite, oldname); err != nil {
+	if err := fs.inj.MaybeError(OpWrite); err != nil {
 		return nil, err
 	}
 	return fs.fs.ReuseForWrite(oldname, newname)
@@ -291,7 +227,7 @@ func (fs *FS) ReuseForWrite(oldname, newname string) (vfs.File, error) {
 
 // MkdirAll implements FS.MkdirAll.
 func (fs *FS) MkdirAll(dir string, perm os.FileMode) error {
-	if err := fs.inj.MaybeError(OpMkdirAll, dir); err != nil {
+	if err := fs.inj.MaybeError(OpWrite); err != nil {
 		return err
 	}
 	return fs.fs.MkdirAll(dir, perm)
@@ -299,7 +235,7 @@ func (fs *FS) MkdirAll(dir string, perm os.FileMode) error {
 
 // Lock implements FS.Lock.
 func (fs *FS) Lock(name string) (io.Closer, error) {
-	if err := fs.inj.MaybeError(OpLock, name); err != nil {
+	if err := fs.inj.MaybeError(OpWrite); err != nil {
 		return nil, err
 	}
 	return fs.fs.Lock(name)
@@ -307,7 +243,7 @@ func (fs *FS) Lock(name string) (io.Closer, error) {
 
 // List implements FS.List.
 func (fs *FS) List(dir string) ([]string, error) {
-	if err := fs.inj.MaybeError(OpList, dir); err != nil {
+	if err := fs.inj.MaybeError(OpRead); err != nil {
 		return nil, err
 	}
 	return fs.fs.List(dir)
@@ -315,7 +251,7 @@ func (fs *FS) List(dir string) ([]string, error) {
 
 // Stat implements FS.Stat.
 func (fs *FS) Stat(name string) (os.FileInfo, error) {
-	if err := fs.inj.MaybeError(OpStat, name); err != nil {
+	if err := fs.inj.MaybeError(OpRead); err != nil {
 		return nil, err
 	}
 	return fs.fs.Stat(name)
@@ -324,7 +260,6 @@ func (fs *FS) Stat(name string) (os.FileInfo, error) {
 // errorFile implements vfs.File. The interface is implemented on the pointer
 // type to allow pointer equality comparisons.
 type errorFile struct {
-	path string
 	file vfs.File
 	inj  Injector
 }
@@ -336,35 +271,35 @@ func (f *errorFile) Close() error {
 }
 
 func (f *errorFile) Read(p []byte) (int, error) {
-	if err := f.inj.MaybeError(OpFileRead, f.path); err != nil {
+	if err := f.inj.MaybeError(OpRead); err != nil {
 		return 0, err
 	}
 	return f.file.Read(p)
 }
 
 func (f *errorFile) ReadAt(p []byte, off int64) (int, error) {
-	if err := f.inj.MaybeError(OpFileReadAt, f.path); err != nil {
+	if err := f.inj.MaybeError(OpRead); err != nil {
 		return 0, err
 	}
 	return f.file.ReadAt(p, off)
 }
 
 func (f *errorFile) Write(p []byte) (int, error) {
-	if err := f.inj.MaybeError(OpFileWrite, f.path); err != nil {
+	if err := f.inj.MaybeError(OpWrite); err != nil {
 		return 0, err
 	}
 	return f.file.Write(p)
 }
 
 func (f *errorFile) Stat() (os.FileInfo, error) {
-	if err := f.inj.MaybeError(OpFileStat, f.path); err != nil {
+	if err := f.inj.MaybeError(OpRead); err != nil {
 		return nil, err
 	}
 	return f.file.Stat()
 }
 
 func (f *errorFile) Sync() error {
-	if err := f.inj.MaybeError(OpFileSync, f.path); err != nil {
+	if err := f.inj.MaybeError(OpWrite); err != nil {
 		return err
 	}
 	return f.file.Sync()
diff --git a/open.go b/open.go
index 8c0444fe..0f9ad3fd 100644
--- a/open.go
+++ b/open.go
@@ -293,23 +293,6 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 	if !d.opts.ReadOnly {
 		// Create an empty .log file.
 		newLogNum := d.mu.versions.getNextFileNum()
-
-		// This logic is slightly different than RocksDB's. Specifically, RocksDB
-		// sets MinUnflushedLogNum to max-recovered-log-num + 1. We set it to the
-		// newLogNum. There should be no difference in using either value.
-		ve.MinUnflushedLogNum = newLogNum
-
-		// Create the manifest with the updated MinUnflushedLogNum before
-		// creating the new log file. If we created the log file first, a
-		// crash before the manifest is synced could leave two WALs with
-		// unclean tails.
-		d.mu.versions.logLock()
-		if err := d.mu.versions.logAndApply(jobID, &ve, newFileMetrics(ve.NewFiles), d.dataDir, func() []compactionInfo {
-			return nil
-		}); err != nil {
-			return nil, err
-		}
-
 		newLogName := base.MakeFilename(opts.FS, d.walDirname, fileTypeLog, newLogNum)
 		d.mu.log.queue = append(d.mu.log.queue, fileInfo{fileNum: newLogNum, fileSize: 0})
 		logFile, err := opts.FS.Create(newLogName)
@@ -335,6 +318,17 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 		d.mu.log.LogWriter = record.NewLogWriter(logFile, newLogNum)
 		d.mu.log.LogWriter.SetMinSyncInterval(d.opts.WALMinSyncInterval)
 		d.mu.versions.metrics.WAL.Files++
+
+		// This logic is slightly different than RocksDB's. Specifically, RocksDB
+		// sets MinUnflushedLogNum to max-recovered-log-num + 1. We set it to the
+		// newLogNum. There should be no difference in using either value.
+		ve.MinUnflushedLogNum = newLogNum
+		d.mu.versions.logLock()
+		if err := d.mu.versions.logAndApply(jobID, &ve, newFileMetrics(ve.NewFiles), d.dataDir, func() []compactionInfo {
+			return nil
+		}); err != nil {
+			return nil, err
+		}
 	}
 	d.updateReadStateLocked(d.opts.DebugCheck)
 
diff --git a/options.go b/options.go
index a453c5fc..85d6682d 100644
--- a/options.go
+++ b/options.go
@@ -438,6 +438,11 @@ type Options struct {
 	// when L0 read-amplification passes the L0CompactionConcurrency threshold.
 	MaxConcurrentCompactions int
 
+	// NumPrevManifest is the number of non-current or older manifests which
+	// we want to keep around for debugging purposes. By default, we're going
+	// to keep one older manifest.
+	NumPrevManifest int
+
 	// ReadOnly indicates that the DB should be opened in read-only mode. Writes
 	// to the DB will return an error, background compactions are disabled, and
 	// the flush that normally occurs after replaying the WAL at startup is
@@ -595,6 +600,10 @@ func (o *Options) EnsureDefaults() *Options {
 	if o.MaxConcurrentCompactions <= 0 {
 		o.MaxConcurrentCompactions = 1
 	}
+	if o.NumPrevManifest <= 0 {
+		o.NumPrevManifest = 1
+	}
+
 	if o.FS == nil {
 		o.FS = vfs.WithDiskHealthChecks(vfs.Default, 5*time.Second,
 			func(name string, duration time.Duration) {


Test patch:
diff --git a/db_test.go b/db_test.go
index d673f6a4..398ae698 100644
--- a/db_test.go
+++ b/db_test.go
@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -837,10 +838,12 @@ func TestFlushEmpty(t *testing.T) {
 }
 
 func TestRollManifest(t *testing.T) {
+	toPreserve := rand.Int31n(5) + 1
 	opts := &Options{
 		MaxManifestFileSize:   1,
 		L0CompactionThreshold: 10,
 		FS:                    vfs.NewMem(),
+		NumPrevManifest:       int(toPreserve),
 	}
 	opts.private.disableAutomaticCompactions = true
 	d, err := Open("", opts)
@@ -868,6 +871,7 @@ func TestRollManifest(t *testing.T) {
 	}
 
 	lastManifestNum := manifestFileNumber()
+	manifestNums := []base.FileNum{lastManifestNum}
 	for i := 0; i < 5; i++ {
 		require.NoError(t, d.Set([]byte("a"), nil, nil))
 		require.NoError(t, d.Flush())
@@ -875,6 +879,8 @@ func TestRollManifest(t *testing.T) {
 		if lastManifestNum == num {
 			t.Fatalf("manifest failed to roll: %d == %d", lastManifestNum, num)
 		}
+
+		manifestNums = append(manifestNums, num)
 		lastManifestNum = num
 
 		expectedCurrent := fmt.Sprintf("MANIFEST-%s\n", lastManifestNum)
@@ -896,7 +902,18 @@ func TestRollManifest(t *testing.T) {
 			manifests = append(manifests, filename)
 		}
 	}
-	expected := []string{fmt.Sprintf("MANIFEST-%s", lastManifestNum)}
+
+	sort.Slice(manifests, func(i, j int) bool {
+		return manifests[i] < manifests[j]
+	})
+
+	var expected []string
+	for i := len(manifestNums) - int(toPreserve) - 1; i < len(manifestNums); i++ {
+		expected = append(
+			expected,
+			fmt.Sprintf("MANIFEST-%s", manifestNums[i]),
+		)
+	}
 	require.EqualValues(t, expected, manifests)
 
 	require.NoError(t, d.Close())
diff --git a/error_test.go b/error_test.go
index 165f5a83..47411449 100644
--- a/error_test.go
+++ b/error_test.go
@@ -335,8 +335,8 @@ func TestDBWALRotationCrash(t *testing.T) {
 	memfs := vfs.NewStrictMem()
 
 	var index int32
-	inj := errorfs.InjectorFunc(func(op errorfs.Op, _ string) error {
-		if op.OpKind() == errorfs.OpKindWrite && atomic.AddInt32(&index, -1) == -1 {
+	inj := errorfs.InjectorFunc(func(op errorfs.Op) error {
+		if op == errorfs.OpWrite && atomic.AddInt32(&index, -1) == -1 {
 			memfs.SetIgnoreSyncs(true)
 		}
 		return nil
diff --git a/internal/metamorphic/meta_test.go b/internal/metamorphic/meta_test.go
index 45ba3812..5d4748de 100644
--- a/internal/metamorphic/meta_test.go
+++ b/internal/metamorphic/meta_test.go
@@ -106,7 +106,7 @@ func testMetaRun(t *testing.T, runDir string, seed uint64) {
 	}
 	// Wrap the filesystem with one that will inject errors into read
 	// operations with *errorRate probability.
-	opts.FS = errorfs.Wrap(opts.FS, errorfs.WithProbability(errorfs.OpKindRead, *errorRate))
+	opts.FS = errorfs.Wrap(opts.FS, errorfs.WithProbability(errorfs.OpRead, *errorRate))
 
 	if opts.WALDir != "" {
 		opts.WALDir = opts.FS.PathJoin(runDir, opts.WALDir)
diff --git a/open_test.go b/open_test.go
index 552dc081..d18605e9 100644
--- a/open_test.go
+++ b/open_test.go
@@ -14,12 +14,10 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync/atomic"
 	"testing"
 
 	"github.com/cockroachdb/errors/oserror"
 	"github.com/cockroachdb/pebble/internal/base"
-	"github.com/cockroachdb/pebble/internal/errorfs"
 	"github.com/cockroachdb/pebble/vfs"
 	"github.com/kr/pretty"
 	"github.com/stretchr/testify/require"
@@ -632,110 +630,6 @@ func TestTwoWALReplayPermissive(t *testing.T) {
 	require.NoError(t, d.Close())
 }
 
-// TestCrashOpenCrashAfterWALCreation tests a database that exits
-// ungracefully, begins recovery, creates the new WAL but promptly exits
-// ungracefully again.
-//
-// This sequence has the potential to be problematic with the strict_wal_tail
-// behavior because the first crash's WAL has an unclean tail. By the time the
-// new WAL is created, the current manifest's MinUnflushedLogNum must be
-// higher than the previous WAL.
-func TestCrashOpenCrashAfterWALCreation(t *testing.T) {
-	fs := vfs.NewStrictMem()
-
-	getLogs := func() (logs []string) {
-		ls, err := fs.List("")
-		require.NoError(t, err)
-		for _, name := range ls {
-			if filepath.Ext(name) == ".log" {
-				logs = append(logs, name)
-			}
-		}
-		return logs
-	}
-
-	{
-		d, err := Open("", &Options{FS: fs})
-		require.NoError(t, err)
-		require.NoError(t, d.Set([]byte("abc"), nil, Sync))
-
-		// Ignore syncs during close to simulate a crash. This will leave the WAL
-		// without an EOF trailer. It won't be an 'unclean tail' yet since the
-		// log file was not recycled, but we'll fix that down below.
-		fs.SetIgnoreSyncs(true)
-		require.NoError(t, d.Close())
-		fs.ResetToSyncedState()
-		fs.SetIgnoreSyncs(false)
-	}
-
-	// There should be one WAL.
-	logs := getLogs()
-	if len(logs) != 1 {
-		t.Fatalf("expected one log file, found %d", len(logs))
-	}
-
-	// The one WAL file doesn't have an EOF trailer, but since it wasn't
-	// recycled it won't have garbage at the end. Rewrite it so that it has
-	// the same contents it currently has, followed by garbage.
-	{
-		f, err := fs.Open(logs[0])
-		require.NoError(t, err)
-		b, err := ioutil.ReadAll(f)
-		require.NoError(t, err)
-		require.NoError(t, f.Close())
-		f, err = fs.Create(logs[0])
-		_, err = f.Write(b)
-		require.NoError(t, err)
-		_, err = f.Write([]byte{0xde, 0xad, 0xbe, 0xef})
-		require.NoError(t, err)
-		require.NoError(t, f.Sync())
-		require.NoError(t, f.Close())
-		dir, err := fs.OpenDir("")
-		require.NoError(t, err)
-		require.NoError(t, dir.Sync())
-		require.NoError(t, dir.Close())
-	}
-
-	// Open the database again (with syncs respected again). Wrap the
-	// filesystem with an errorfs that will turn off syncs after a new .log
-	// file is created and after a subsequent directory sync occurs. This
-	// simulates a crash after the new log file is created and synced.
-	{
-		var atomicWALCreated, atomicDirSynced uint32
-		d, err := Open("", &Options{
-			FS: errorfs.Wrap(fs, errorfs.InjectorFunc(func(op errorfs.Op, path string) error {
-				if atomic.LoadUint32(&atomicDirSynced) == 1 {
-					fs.SetIgnoreSyncs(true)
-				}
-				if op == errorfs.OpCreate && filepath.Ext(path) == ".log" {
-					atomic.StoreUint32(&atomicWALCreated, 1)
-				}
-				// Record when there's a sync of the data directory after the
-				// WAL was created. The data directory will have an empty
-				// path because that's what we passed into Open.
-				if op == errorfs.OpFileSync && path == "" && atomic.LoadUint32(&atomicWALCreated) == 1 {
-					atomic.StoreUint32(&atomicDirSynced, 1)
-				}
-				return nil
-			})),
-		})
-		require.NoError(t, err)
-		require.NoError(t, d.Close())
-	}
-
-	fs.ResetToSyncedState()
-	fs.SetIgnoreSyncs(false)
-
-	if n := len(getLogs()); n != 2 {
-		t.Fatalf("expected two logs, found %d\n", n)
-	}
-
-	// Finally, open the database with syncs enabled.
-	d, err := Open("", &Options{FS: fs})
-	require.NoError(t, err)
-	require.NoError(t, d.Close())
-}
-
 // TestOpenWALReplayReadOnlySeqNums tests opening a database:
 // * in read-only mode
 // * with multiple unflushed log files that must replayed
diff --git a/testdata/checkpoint b/testdata/checkpoint
index ce018e2f..d0b64aa6 100644
--- a/testdata/checkpoint
+++ b/testdata/checkpoint
@@ -10,9 +10,9 @@ sync: db/CURRENT.000001.dbtmp
 close: db/CURRENT.000001.dbtmp
 rename: db/CURRENT.000001.dbtmp -> db/CURRENT
 sync: db
-sync: db/MANIFEST-000001
 create: db/000002.log
 sync: db
+sync: db/MANIFEST-000001
 create: db/OPTIONS-000003
 sync: db/OPTIONS-000003
 close: db/OPTIONS-000003
diff --git a/testdata/cleaner b/testdata/cleaner
index eb61ca39..67ff6700 100644
--- a/testdata/cleaner
+++ b/testdata/cleaner
@@ -12,9 +12,9 @@ sync: db/CURRENT.000001.dbtmp
 close: db/CURRENT.000001.dbtmp
 rename: db/CURRENT.000001.dbtmp -> db/CURRENT
 sync: db
-sync: db/MANIFEST-000001
 create: wal/000002.log
 sync: wal
+sync: db/MANIFEST-000001
 create: db/OPTIONS-000003
 sync: db/OPTIONS-000003
 close: db/OPTIONS-000003
diff --git a/testdata/event_listener b/testdata/event_listener
index 55a9d783..4d821a06 100644
--- a/testdata/event_listener
+++ b/testdata/event_listener
@@ -13,6 +13,9 @@ close: db/CURRENT.000001.dbtmp
 rename: db/CURRENT.000001.dbtmp -> db/CURRENT
 sync: db
 [JOB 1] MANIFEST created 000001
+create: wal/000002.log
+sync: wal
+[JOB 1] WAL created 000002
 create: db/MANIFEST-000003
 close: db/MANIFEST-000001
 sync: db/MANIFEST-000003
@@ -22,14 +25,10 @@ close: db/CURRENT.000003.dbtmp
 rename: db/CURRENT.000003.dbtmp -> db/CURRENT
 sync: db
 [JOB 1] MANIFEST created 000003
-create: wal/000002.log
-sync: wal
-[JOB 1] WAL created 000002
 create: db/OPTIONS-000004
 sync: db/OPTIONS-000004
 close: db/OPTIONS-000004
 sync: db
-[JOB 1] MANIFEST deleted 000001
 
 flush
 ----
@@ -55,7 +54,7 @@ rename: db/CURRENT.000007.dbtmp -> db/CURRENT
 sync: db
 [JOB 3] MANIFEST created 000007
 [JOB 3] flushed 1 memtable to L0 [000006] (770 B), in 1.0s, output rate 770 B/s
-[JOB 3] MANIFEST deleted 000003
+[JOB 3] MANIFEST deleted 000001
 
 compact
 ----
@@ -81,7 +80,7 @@ rename: db/CURRENT.000010.dbtmp -> db/CURRENT
 sync: db
 [JOB 5] MANIFEST created 000010
 [JOB 5] flushed 1 memtable to L0 [000009] (770 B), in 1.0s, output rate 770 B/s
-[JOB 5] MANIFEST deleted 000007
+[JOB 5] MANIFEST deleted 000003
 [JOB 6] compacting(default) L0 [000006 000009] (1.5 K) + L6 [] (0 B)
 create: db/000011.sst
 [JOB 6] compacting: sstable created 000011
@@ -100,7 +99,7 @@ sync: db
 [JOB 6] compacted(default) L0 [000006 000009] (1.5 K) + L6 [] (0 B) -> L6 [000011] (770 B), in 1.0s, output rate 770 B/s
 [JOB 6] sstable deleted 000006
 [JOB 6] sstable deleted 000009
-[JOB 6] MANIFEST deleted 000010
+[JOB 6] MANIFEST deleted 000007
 
 disable-file-deletions
 ----
@@ -132,7 +131,7 @@ sync: db
 
 enable-file-deletions
 ----
-[JOB 9] MANIFEST deleted 000012
+[JOB 9] MANIFEST deleted 000010
 
 ingest
 ----
@@ -148,7 +147,7 @@ close: db/CURRENT.000017.dbtmp
 rename: db/CURRENT.000017.dbtmp -> db/CURRENT
 sync: db
 [JOB 10] MANIFEST created 000017
-[JOB 10] MANIFEST deleted 000015
+[JOB 10] MANIFEST deleted 000012
 [JOB 10] ingested L0:000016 (825 B)
 
 metrics