Repository: cockroachdb/pebble

Issue:
manifest: version edit decode misinterprets ErrUnexpectedEOF
During version edit decoding, we read from an `io.Reader` backed by the `record` package. If a record in the MANIFEST is spread across multiple chunks, this reader handles reading new chunks as the caller pulls from the reader. It's possible for a record that is split across multiple chunks to be only partially persisted if it was in the process of being written when the process exited. When this happens, the underlying reader will surface an `io.ErrUnexpectedEOF`.

During version edit decoding, when we expect to read _n_ bytes from a record and find fewer than _n_ bytes in the payload, this gets surfaced from `io.ReadFull` as an `io.ErrUnexpectedEOF`. This case should be considered corruption, because the payload is improper.

Currently, because these scenarios both use `io.ErrUnexpectedEOF` these cases are indistinguishable. If the `io.ErrUnexpectedEOF` was surfaced by the underlying record reader, we want to treat this as a logical EOF, not an error.


Jira issue: PEBBLE-404

Hints from issue comments:

Solution patch:
diff --git a/internal/manifest/version_edit.go b/internal/manifest/version_edit.go
index baf151fa..b8c6bb90 100644
--- a/internal/manifest/version_edit.go
+++ b/internal/manifest/version_edit.go
@@ -25,8 +25,6 @@ import (
 // TODO(peter): describe the MANIFEST file format, independently of the C++
 // project.
 
-var errCorruptManifest = base.CorruptionErrorf("pebble: corrupt manifest")
-
 type byteReader interface {
 	io.ByteReader
 	io.Reader
@@ -548,7 +546,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 			return base.CorruptionErrorf("column families are not supported")
 
 		default:
-			return errCorruptManifest
+			return base.CorruptionErrorf("MANIFEST: unknown tag: %d", tag)
 		}
 	}
 	return nil
@@ -842,7 +840,7 @@ func (d versionEditDecoder) readBytes() ([]byte, error) {
 	_, err = io.ReadFull(d, s)
 	if err != nil {
 		if err == io.ErrUnexpectedEOF {
-			return nil, errCorruptManifest
+			return nil, base.CorruptionErrorf("pebble: corrupt manifest: failed to read %d bytes", n)
 		}
 		return nil, err
 	}
@@ -855,7 +853,7 @@ func (d versionEditDecoder) readLevel() (int, error) {
 		return 0, err
 	}
 	if u >= NumLevels {
-		return 0, errCorruptManifest
+		return 0, base.CorruptionErrorf("pebble: corrupt manifest: level %d >= %d", u, NumLevels)
 	}
 	return int(u), nil
 }
@@ -871,8 +869,8 @@ func (d versionEditDecoder) readFileNum() (base.FileNum, error) {
 func (d versionEditDecoder) readUvarint() (uint64, error) {
 	u, err := binary.ReadUvarint(d)
 	if err != nil {
-		if err == io.EOF {
-			return 0, errCorruptManifest
+		if err == io.EOF || err == io.ErrUnexpectedEOF {
+			return 0, base.CorruptionErrorf("pebble: corrupt manifest: failed to read uvarint")
 		}
 		return 0, err
 	}
diff --git a/open.go b/open.go
index 84791c8e..db60a605 100644
--- a/open.go
+++ b/open.go
@@ -950,20 +950,29 @@ func (d *DB) replayWAL(
 			// surfaces record.ErrInvalidChunk or record.ErrZeroedChunk. These
 			// errors are always indicative of corruption and data loss.
 			//
-			// Otherwise, the reader surfaces io.ErrUnexpectedEOF indicating that
-			// the WAL terminated uncleanly and ambiguously. If the WAL is the
-			// most recent logical WAL, the caller passes in (strictWALTail=false),
-			// indicating we should tolerate the unclean ending. If the WAL is an
-			// older WAL, the caller passes in (strictWALTail=true), indicating that
-			// the WAL should have been closed cleanly, and we should interpret
-			// the `io.ErrUnexpectedEOF` as corruption and stop recovery.
+			// Otherwise, the reader surfaces record.ErrUnexpectedEOF indicating
+			// that the WAL terminated uncleanly and ambiguously. If the WAL is
+			// the most recent logical WAL, the caller passes in
+			// (strictWALTail=false), indicating we should tolerate the unclean
+			// ending. If the WAL is an older WAL, the caller passes in
+			// (strictWALTail=true), indicating that the WAL should have been
+			// closed cleanly, and we should interpret the
+			// `record.ErrUnexpectedEOF` as corruption and stop recovery.
 			if errors.Is(err, io.EOF) {
 				break
-			} else if errors.Is(err, io.ErrUnexpectedEOF) && !strictWALTail {
+			} else if errors.Is(err, record.ErrUnexpectedEOF) && !strictWALTail {
 				break
-			} else if errors.Is(err, record.ErrInvalidChunk) || errors.Is(err, record.ErrZeroedChunk) {
-				// If a read-ahead returns one of these errors, they should be marked with corruption.
-				// Other I/O related errors should not be marked with corruption and simply returned.
+			} else if (errors.Is(err, record.ErrUnexpectedEOF) && strictWALTail) ||
+				errors.Is(err, record.ErrInvalidChunk) || errors.Is(err, record.ErrZeroedChunk) {
+				// If a read-ahead returns record.ErrInvalidChunk or
+				// record.ErrZeroedChunk, then there's definitively corruption.
+				//
+				// If strictWALTail=true, then record.ErrUnexpectedEOF should
+				// also be considered corruption because the strictWALTail
+				// indicates we expect a clean end to the WAL.
+				//
+				// Other I/O related errors should not be marked with corruption
+				// and simply returned.
 				err = errors.Mark(err, ErrCorruption)
 			}
 
diff --git a/record/record.go b/record/record.go
index 65f840e8..c3dd20fc 100644
--- a/record/record.go
+++ b/record/record.go
@@ -214,13 +214,28 @@ var (
 	// header, length, or checksum. This usually occurs when a log is recycled,
 	// but can also occur due to corruption.
 	ErrInvalidChunk = errors.New("pebble/record: invalid chunk")
+
+	// ErrUnexpectedEOF is returned if a log file ends unexpectedly. It
+	// indicates the unexpected end of the log file or an in-progress record
+	// envelope itself. ErrUnexpectedEOF may be returned by Reader when it
+	// encounters an invalid chunk but observes no evidence that the invalid
+	// chunk is caused by corruption (i.e., no future chunk indicates that
+	// offset should be valid and durably synced.)
+	//
+	// This error is defined separately from io.ErrUnexpectedEOF to disambiguate
+	// this case from from the case of an unexpected end of the record's payload
+	// while decoding at a higher-level (eg, version edit decoding). If a
+	// higher-level decoding routine returns record.ErrUnexpectedEOF, it
+	// unambiguously indicates that the log file itself ended unexpectedly. The
+	// record.Reader will never return io.ErrUnexpectedEOF, just record.ErrUnexpectedEOF.
+	ErrUnexpectedEOF = errors.New("pebble/record: unexpected EOF")
 )
 
 // IsInvalidRecord returns true if the error matches one of the error types
 // returned for invalid records. These are treated in a way similar to io.EOF
 // in recovery code.
 func IsInvalidRecord(err error) bool {
-	return err == ErrZeroedChunk || err == ErrInvalidChunk || err == io.ErrUnexpectedEOF
+	return err == ErrZeroedChunk || err == ErrInvalidChunk || err == ErrUnexpectedEOF
 }
 
 // Reader reads records from an underlying io.Reader.
@@ -268,7 +283,7 @@ func NewReader(r io.Reader, logNum base.DiskFileNum) *Reader {
 		logNum:   uint32(logNum),
 		blockNum: -1,
 		// invalidOffset is initialized as MaxUint64 so that reading ahead
-		// with the old chunk wire formats results in io.ErrUnexpectedEOF.
+		// with the old chunk wire formats results in ErrUnexpectedEOF.
 		invalidOffset: math.MaxUint64,
 	}
 }
@@ -385,7 +400,7 @@ func (r *Reader) nextChunk(wantFirst bool) error {
 		if err != nil && err != io.ErrUnexpectedEOF {
 			if err == io.EOF && !wantFirst {
 				r.invalidOffset = uint64(r.blockNum)*blockSize + uint64(r.begin)
-				return io.ErrUnexpectedEOF
+				return ErrUnexpectedEOF
 			}
 			return err
 		}
@@ -443,20 +458,19 @@ func (r *Reader) readAheadForCorruption() error {
 		}
 
 		if errors.Is(err, io.EOF) {
-			// io.ErrUnexpectedEOF is returned instead of
-			// io.EOF because io library functions clear
-			// an error when it is io.EOF. io.ErrUnexpectedEOF
-			// is returned so that the error is not cleared
-			// when the io library makes calls to Reader.Read().
+			// ErrUnexpectedEOF is returned instead of io.EOF because io library
+			// functions clear an error when it is io.EOF. ErrUnexpectedEOF is
+			// returned so that the error is not cleared when the io library
+			// makes calls to Reader.Read().
 			//
-			// Since no sync offset was found to indicate that the
-			// invalid chunk should have been valid, the chunk represents
-			// an abrupt, unclean termination of the logical log. This
-			// abrupt end of file represented by io.ErrUnexpectedEOF.
+			// Since no sync offset was found to indicate that the invalid chunk
+			// should have been valid, the chunk represents an abrupt, unclean
+			// termination of the logical log. This abrupt end of file
+			// represented by ErrUnexpectedEOF.
 			if r.loggerForTesting != nil {
-				r.loggerForTesting.logf("\tEncountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.\n")
+				r.loggerForTesting.logf("\tEncountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.\n")
 			}
-			return io.ErrUnexpectedEOF
+			return ErrUnexpectedEOF
 		}
 		// The last block of a log can be less than 32KiB, which is
 		// the length of r.buf. Thus, we should still parse the data in



Test patch:
diff --git a/compaction_test.go b/compaction_test.go
index 3c70e106..8c6020a2 100644
--- a/compaction_test.go
+++ b/compaction_test.go
@@ -1350,6 +1350,17 @@ func TestCompactionDeleteOnlyHints(t *testing.T) {
 	datadriven.RunTest(t, "testdata/compaction_delete_only_hints",
 		func(t *testing.T, td *datadriven.TestData) string {
 			switch td.Cmd {
+			case "batch":
+				b := d.NewBatch()
+				err := runBatchDefineCmd(td, b)
+				if err != nil {
+					return err.Error()
+				}
+				if err = b.Commit(Sync); err != nil {
+					return err.Error()
+				}
+				return ""
+
 			case "define":
 				opts, err = reset()
 				if err != nil {
@@ -1364,6 +1375,12 @@ func TestCompactionDeleteOnlyHints(t *testing.T) {
 				d.mu.Unlock()
 				return s
 
+			case "flush":
+				if err := d.Flush(); err != nil {
+					return err.Error()
+				}
+				return runLSMCmd(td, d)
+
 			case "force-set-hints":
 				d.mu.Lock()
 				defer d.mu.Unlock()
@@ -1514,6 +1531,13 @@ func TestCompactionDeleteOnlyHints(t *testing.T) {
 				}
 				return ""
 
+			case "snapshot":
+				// NB: It's okay that we're letting the snapshot out of scope
+				// without closing it; the close snapshot command will pull the
+				// relevant seqnum off the DB to close it.
+				s := d.NewSnapshot()
+				return fmt.Sprintf("snapshot seqnum: %d", s.seqNum)
+
 			case "ingest":
 				if err = runBuildCmd(td, d, d.opts.FS); err != nil {
 					return err.Error()
diff --git a/record/record_test.go b/record/record_test.go
index d38d830a..e9184719 100644
--- a/record/record_test.go
+++ b/record/record_test.go
@@ -437,7 +437,7 @@ func TestCorruptBlock(t *testing.T) {
 	if err == nil {
 		t.Fatal("Expected a checksum mismatch error, got nil")
 	}
-	if err != io.ErrUnexpectedEOF {
+	if !errors.Is(err, ErrUnexpectedEOF) {
 		t.Fatalf("Unexpected error returned: %v", err)
 	}
 }
@@ -582,7 +582,7 @@ func TestSeekRecordEndOfFile(t *testing.T) {
 	if err == nil {
 		t.Fatalf("Seek past the end of a file didn't cause an error")
 	}
-	if err != io.ErrUnexpectedEOF {
+	if err != ErrUnexpectedEOF {
 		t.Fatalf("Seeking past EOF raised unexpected error: %v", err)
 	}
 }
@@ -669,8 +669,8 @@ func TestInvalidLogNum(t *testing.T) {
 
 	{
 		r := NewReader(bytes.NewReader(buf.Bytes()), 2)
-		if _, err := r.Next(); err != io.ErrUnexpectedEOF {
-			t.Fatalf("expected %s, but found %s\n", io.ErrUnexpectedEOF, err)
+		if _, err := r.Next(); !errors.Is(err, ErrUnexpectedEOF) {
+			t.Fatalf("expected %s, but found %s\n", ErrUnexpectedEOF, err)
 		}
 	}
 }
@@ -753,7 +753,7 @@ func TestRecycleLog(t *testing.T) {
 			rr, err := r.Next()
 			if err != nil {
 				// If we limited output then an EOF, zeroed, or invalid chunk is expected.
-				if limitedBuf.limit < 0 && (err == io.EOF || err == io.ErrUnexpectedEOF || err == ErrZeroedChunk || err == ErrInvalidChunk) {
+				if limitedBuf.limit < 0 && (err == io.EOF || IsInvalidRecord(err)) {
 					break
 				}
 				t.Fatalf("%d/%d: %v", i, j, err)
@@ -761,7 +761,7 @@ func TestRecycleLog(t *testing.T) {
 			x, err := io.ReadAll(rr)
 			if err != nil {
 				// If we limited output then an EOF, zeroed, or invalid chunk is expected.
-				if limitedBuf.limit < 0 && (err == io.EOF || err == io.ErrUnexpectedEOF || err == ErrZeroedChunk || err == ErrInvalidChunk) {
+				if limitedBuf.limit < 0 && (err == io.EOF || IsInvalidRecord(err)) {
 					break
 				}
 				t.Fatalf("%d/%d: %v", i, j, err)
@@ -770,7 +770,7 @@ func TestRecycleLog(t *testing.T) {
 				t.Fatalf("%d/%d: expected record %d, but found %d", i, j, sizes[j], len(x))
 			}
 		}
-		if _, err := r.Next(); err != io.EOF && err != io.ErrUnexpectedEOF && err != ErrZeroedChunk && err != ErrInvalidChunk {
+		if _, err := r.Next(); err != io.EOF && err != ErrUnexpectedEOF && err != ErrZeroedChunk && err != ErrInvalidChunk {
 			t.Fatalf("%d: expected EOF, but found %v", i, err)
 		}
 	}
@@ -789,7 +789,7 @@ func TestTruncatedLog(t *testing.T) {
 	rr, err := r.Next()
 	require.NoError(t, err)
 	_, err = io.ReadAll(rr)
-	require.EqualValues(t, err, io.ErrUnexpectedEOF)
+	require.EqualValues(t, err, ErrUnexpectedEOF)
 }
 
 func TestRecycleLogWithPartialBlock(t *testing.T) {
@@ -881,7 +881,7 @@ func TestRecycleLogWithPartialRecord(t *testing.T) {
 	require.NoError(t, err)
 
 	_, err = io.ReadAll(rr)
-	require.Equal(t, err, io.ErrUnexpectedEOF)
+	require.True(t, errors.Is(err, ErrUnexpectedEOF))
 }
 
 type readerLogger struct {
diff --git a/record/testdata/walSync b/record/testdata/walSync
index e335d6f0..45451333 100644
--- a/record/testdata/walSync
+++ b/record/testdata/walSync
@@ -47,8 +47,8 @@ read
 ----
 Starting read ahead for corruption. Block corrupted 0.
 Read block 1 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 1
 bytes read: 0
 
@@ -66,8 +66,8 @@ read
 ----
 Starting read ahead for corruption. Block corrupted 0.
 Read block 1 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 1
 bytes read: 0
 
@@ -84,8 +84,8 @@ Read block 1 with 19 bytes
 	Block 1: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 1 at offset 0 (expected 1, got 2)
 Read block 2 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 2
 bytes read: 0
 
@@ -121,8 +121,8 @@ Read block 2 with 19 bytes
 	Block 2: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 2 at offset 0 (expected 1, got 2)
 Read block 3 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 3
 bytes read: 0
 
@@ -165,8 +165,8 @@ Read block 2 with 19 bytes
 	Block 2: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 2 at offset 0 (expected 1, got 2)
 Read block 3 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 3
 bytes read: 0
 
@@ -308,8 +308,8 @@ Read block 2 with 19 bytes
 	Block 2: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 2 at offset 0 (expected 1, got 2)
 Read block 3 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 3
 bytes read: 0
 
@@ -376,8 +376,8 @@ Read block 4 with 19 bytes
 	Block 4: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 4 at offset 0 (expected 1, got 2)
 Read block 5 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 5
 bytes read: 0
 
@@ -555,8 +555,8 @@ Read block 4 with 19 bytes
 	Block 4: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 4 at offset 0 (expected 1, got 2)
 Read block 5 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 5
 bytes read: 32749
 
@@ -625,8 +625,8 @@ read
 ----
 Starting read ahead for corruption. Block corrupted 0.
 Read block 1 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 1
 bytes read: 0
 
@@ -643,8 +643,8 @@ Read block 1 with 19 bytes
 	Block 1: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 1 at offset 0 (expected 1, got 2)
 Read block 2 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 2
 bytes read: 0
 
@@ -665,8 +665,8 @@ Read block 2 with 19 bytes
 	Block 2: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 2 at offset 0 (expected 1, got 2)
 Read block 3 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 3
 bytes read: 0
 
@@ -688,8 +688,8 @@ Read block 2 with 19 bytes
 	Block 2: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 2 at offset 0 (expected 1, got 2)
 Read block 3 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 3
 bytes read: 0
 
@@ -754,8 +754,8 @@ Read block 2 with 19 bytes
 	Block 2: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 2 at offset 0 (expected 1, got 2)
 Read block 3 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 3
 bytes read: 0
 
@@ -789,8 +789,8 @@ Read block 4 with 19 bytes
 	Block 4: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 4 at offset 0 (expected 1, got 2)
 Read block 5 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 5
 bytes read: 0
 
@@ -855,8 +855,8 @@ Read block 4 with 19 bytes
 	Block 4: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 4 at offset 0 (expected 1, got 2)
 Read block 5 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 5
 bytes read: 32749
 
@@ -876,8 +876,8 @@ read
 ----
 Starting read ahead for corruption. Block corrupted 0.
 Read block 1 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 1
 bytes read: 0
 
@@ -894,8 +894,8 @@ Read block 1 with 19 bytes
 	Block 1: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 1 at offset 0 (expected 1, got 2)
 Read block 2 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 2
 bytes read: 0
 
@@ -916,8 +916,8 @@ Read block 2 with 19 bytes
 	Block 2: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 2 at offset 0 (expected 1, got 2)
 Read block 3 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 3
 bytes read: 0
 
@@ -939,8 +939,8 @@ Read block 2 with 19 bytes
 	Block 2: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 2 at offset 0 (expected 1, got 2)
 Read block 3 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 3
 bytes read: 0
 
@@ -1005,8 +1005,8 @@ Read block 2 with 19 bytes
 	Block 2: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 2 at offset 0 (expected 1, got 2)
 Read block 3 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 3
 bytes read: 0
 
@@ -1039,8 +1039,8 @@ Read block 4 with 19 bytes
 	Block 4: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 4 at offset 0 (expected 1, got 2)
 Read block 5 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 5
 bytes read: 0
 
@@ -1104,8 +1104,8 @@ Read block 4 with 19 bytes
 	Block 4: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 4 at offset 0 (expected 1, got 2)
 Read block 5 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 5
 bytes read: 32749
 
@@ -1128,8 +1128,8 @@ Read block 1 with 19 bytes
 	Block 1: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 1 at offset 0 (expected 1, got 2)
 Read block 2 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 2
 bytes read: 32730
 
@@ -1154,8 +1154,8 @@ Read block 2 with 19 bytes
 	Block 2: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 2 at offset 0 (expected 1, got 2)
 Read block 3 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 3
 bytes read: 0
 
@@ -1178,8 +1178,8 @@ Read block 2 with 19 bytes
 	Block 2: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 2 at offset 0 (expected 1, got 2)
 Read block 3 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 3
 bytes read: 32730
 
@@ -1265,8 +1265,8 @@ Read block 2 with 19 bytes
 	Block 2: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 2 at offset 0 (expected 1, got 2)
 Read block 3 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 3
 bytes read: 0
 
@@ -1326,8 +1326,8 @@ Read block 4 with 19 bytes
 	Block 4: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 4 at offset 0 (expected 1, got 2)
 Read block 5 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 5
 bytes read: 32750
 
@@ -1363,8 +1363,8 @@ read
 ----
 Starting read ahead for corruption. Block corrupted 0.
 Read block 1 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 1
 bytes read: 0
 
@@ -1382,8 +1382,8 @@ Read block 1 with 19 bytes
 	Block 1: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 1 at offset 0 (expected 1, got 2)
 Read block 2 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 2
 bytes read: 0
 
@@ -1405,8 +1405,8 @@ Read block 2 with 19 bytes
 	Block 2: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 2 at offset 0 (expected 1, got 2)
 Read block 3 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 3
 bytes read: 0
 
@@ -1428,8 +1428,8 @@ Read block 2 with 19 bytes
 	Block 2: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 2 at offset 0 (expected 1, got 2)
 Read block 3 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 3
 bytes read: 0
 
@@ -1494,8 +1494,8 @@ Read block 2 with 19 bytes
 	Block 2: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 2 at offset 0 (expected 1, got 2)
 Read block 3 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 3
 bytes read: 0
 
@@ -1528,8 +1528,8 @@ Read block 4 with 19 bytes
 	Block 4: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 4 at offset 0 (expected 1, got 2)
 Read block 5 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 5
 bytes read: 0
 
@@ -1593,8 +1593,8 @@ Read block 5 with 19 bytes
 	Block 5: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 5 at offset 0 (expected 1, got 2)
 Read block 6 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 6
 bytes read: 32750
 
@@ -1613,8 +1613,8 @@ read
 ----
 Starting read ahead for corruption. Block corrupted 0.
 Read block 1 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 1
 bytes read: 0
 
@@ -1631,8 +1631,8 @@ Read block 1 with 19 bytes
 	Block 1: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 1 at offset 0 (expected 1, got 2)
 Read block 2 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 2
 bytes read: 0
 
@@ -1653,8 +1653,8 @@ Read block 2 with 19 bytes
 	Block 2: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 2 at offset 0 (expected 1, got 2)
 Read block 3 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 3
 bytes read: 0
 
@@ -1675,8 +1675,8 @@ Read block 2 with 19 bytes
 	Block 2: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 2 at offset 0 (expected 1, got 2)
 Read block 3 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 3
 bytes read: 0
 
@@ -1741,8 +1741,8 @@ Read block 2 with 19 bytes
 	Block 2: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 2 at offset 0 (expected 1, got 2)
 Read block 3 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 3
 bytes read: 0
 
@@ -1775,8 +1775,8 @@ Read block 4 with 19 bytes
 	Block 4: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 4 at offset 0 (expected 1, got 2)
 Read block 5 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 5
 bytes read: 0
 
@@ -1834,8 +1834,8 @@ Read block 4 with 19 bytes
 	Block 4: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 4 at offset 0 (expected 1, got 2)
 Read block 5 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 5
 bytes read: 32749
 
@@ -1877,8 +1877,8 @@ Read block 1 with 19 bytes
 	Block 1: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 1 at offset 0 (expected 1, got 2)
 Read block 2 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 2
 bytes read: 32738
 
@@ -1903,8 +1903,8 @@ Read block 2 with 19 bytes
 	Block 2: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 2 at offset 0 (expected 1, got 2)
 Read block 3 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 3
 bytes read: 32738
 
@@ -1991,8 +1991,8 @@ Read block 2 with 19 bytes
 	Block 2: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 2 at offset 0 (expected 1, got 2)
 Read block 3 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 3
 bytes read: 1
 
@@ -2053,7 +2053,7 @@ Read block 4 with 19 bytes
 	Block 4: Processing chunk at offset 0, checksum=0, length=0, encoding=9
 	Mismatch log number in block 4 at offset 0 (expected 1, got 2)
 Read block 5 with 0 bytes
-	Encountered io.EOF; returning io.ErrUnexpectedEOF since no sync offset found.
-error reading next: unexpected EOF
+	Encountered io.EOF; returning ErrUnexpectedEOF since no sync offset found.
+error reading next: pebble/record: unexpected EOF
 final blockNum: 5
 bytes read: 32750
diff --git a/testdata/compaction_delete_only_hints b/testdata/compaction_delete_only_hints
index f6596e78..7fe7f993 100644
--- a/testdata/compaction_delete_only_hints
+++ b/testdata/compaction_delete_only_hints
@@ -702,3 +702,56 @@ o: (o, .)
 get-hints
 ----
 (none)
+
+
+# Reset the database.
+
+reset
+----
+
+# Simulate an ingestion of a CockroachDB snapshot sstable with a range delete and a
+# range key delete covering the keyspace.
+
+ingest ext
+set c c
+set d d
+set e e
+set f f
+set g g
+del-range c h
+range-key-del c h
+----
+OK
+
+describe-lsm
+----
+L6:
+  000004:[c#10,RANGEKEYDEL-h#inf,RANGEDEL]
+
+snapshot
+----
+snapshot seqnum: 11
+
+# Simulate a replica removal of the previously ingested snapshot through a range
+# delete and range key delete.
+
+batch
+del-range c h
+range-key-del c h
+----
+
+flush
+----
+L0.0:
+  000006:[c#12,RANGEKEYDEL-h#inf,RANGEDEL]
+L6:
+  000004:[c#10,RANGEKEYDEL-h#inf,RANGEDEL]
+
+get-hints
+----
+L0.000006 c-h seqnums(tombstone=11-12, file-smallest=10, type=point-and-range-key)
+
+close-snapshot
+11
+----
+[JOB 100] compacted(delete-only) L6 [000004] (994B) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s
diff --git a/version_set_test.go b/version_set_test.go
index c85228b9..7453eda5 100644
--- a/version_set_test.go
+++ b/version_set_test.go
@@ -5,6 +5,7 @@
 package pebble
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io"
@@ -14,6 +15,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
 
 	"github.com/cockroachdb/datadriven"
@@ -25,6 +27,7 @@ import (
 	"github.com/cockroachdb/pebble/sstable"
 	"github.com/cockroachdb/pebble/vfs"
 	"github.com/cockroachdb/pebble/vfs/atomicfs"
+	"github.com/cockroachdb/pebble/vfs/errorfs"
 	"github.com/stretchr/testify/require"
 )
 
@@ -314,3 +317,78 @@ func TestVersionSetSeqNums(t *testing.T) {
 	// logSeqNum is always one greater than the last assigned sequence number.
 	require.Equal(t, d.mu.versions.logSeqNum.Load(), lastSeqNum+1)
 }
+
+// TestCrashDuringManifestWrite_LargeKeys tests a crash mid-manifest write. It
+// uses randomly-sized keys with a very high max in order to test version edits
+// of variable sizes. Large version edits may be broken into multiple 'chunks'
+// across multiple 32KiB blocks within the record package's encoding. There have
+// previously been issues specifically decoding these multi-block version edits.
+func TestCrashDuringManifestWrite_LargeKeys(t *testing.T) {
+	seed := rand.Uint64()
+	t.Logf("seed: %d", seed)
+	rng := rand.New(rand.NewPCG(seed, 0))
+
+	// crashClone is nil until a clone of the memFS is constructed, where the
+	// clone will lose 50% of the unsynced data. Each iteration constructs one
+	// clone at a random time, and the DB keeps setting values until the clone
+	// is created. Then a new DB is opened with the cloned memFS.
+	var crashClone atomic.Pointer[vfs.MemFS]
+	makeFS := func(iter uint64) vfs.FS {
+		memFS := vfs.NewCrashableMem()
+		return errorfs.Wrap(memFS, errorfs.InjectorFunc(func(op errorfs.Op) error {
+			if crashClone.Load() != nil || op.Kind != errorfs.OpFileWrite {
+				return nil
+			}
+			typ, _, ok := base.ParseFilename(memFS, memFS.PathBase(op.Path))
+			if !ok || typ != base.FileTypeManifest {
+				return nil
+			}
+			if rng.IntN(5) == 0 {
+				crashClone.Store(memFS.CrashClone(vfs.CrashCloneCfg{
+					UnsyncedDataPercent: 50,
+					RNG:                 rand.New(rand.NewPCG(seed, iter)),
+				}))
+			}
+			return nil
+		}))
+	}
+
+	opts := &Options{Logger: testLogger{t: t}}
+	lel := MakeLoggingEventListener(opts.Logger)
+	opts.EventListener = &lel
+
+	k := slices.Concat([]byte("averyl"), bytes.Repeat([]byte{'o'}, rng.IntN(100000)), []byte("ngkey"))
+	baseLen := len(k)
+	newKey := func(i int) []byte {
+		return append(k[:baseLen], fmt.Sprintf("%10d", i)...)
+	}
+
+	const numIterations = 10
+	var keyIndex int
+	for i := 0; i < numIterations; i++ {
+		func() {
+			crashClone.Store(nil)
+
+			opts.FS = makeFS(uint64(i))
+			d, err := Open("foo", opts)
+			require.NoError(t, err)
+			func() {
+				defer func() { require.NoError(t, d.Close()) }()
+				for j := 0; crashClone.Load() == nil; j++ {
+					keyIndex++
+					require.NoError(t, d.Set(newKey(keyIndex), []byte("value"), Sync))
+					if j%10 == 0 {
+						_, err := d.AsyncFlush()
+						require.NoError(t, err)
+					}
+				}
+				require.NotNil(t, crashClone.Load())
+			}()
+
+			opts.FS = crashClone.Load()
+			d, err = Open("foo", opts)
+			require.NoError(t, err)
+			require.NoError(t, d.Close())
+		}()
+	}
+}
diff --git a/wal/testdata/reader b/wal/testdata/reader
index 3256c6ac..bc2fe6f6 100644
--- a/wal/testdata/reader
+++ b/wal/testdata/reader
@@ -90,7 +90,7 @@ r.NextRecord() = (rr, (000002.log: 111), <nil>)
 r.NextRecord() = (rr, (000002.log: 272), <nil>)
   io.ReadAll(rr) = ("2a0000000000000001000000ec8367c42ebf0ffad5c57ece37b18559ba95ad78... <64000-byte record>", <nil>)
   BatchHeader: [seqNum=42,count=1]
-r.NextRecord() = (rr, (000002.log: 64294), unexpected EOF)
+r.NextRecord() = (rr, (000002.log: 64294), pebble/record: unexpected EOF)
 
 # Test a typical failure scenario. Start off with a recycled log file (000003)
 # that would be on the primary device. It closes "unclean" because we're unable
@@ -253,7 +253,7 @@ r.NextRecord() = (rr, (000005-001.log: 55), 6022 from previous files, <nil>)
 r.NextRecord() = (rr, (000005-001.log: 482), 6022 from previous files, <nil>)
   io.ReadAll(rr) = ("12750100000000001d0000007575c6296b096226e5e78b9760aa7c2ecfa913b6... <199-byte record>", <nil>)
   BatchHeader: [seqNum=95506,count=29]
-r.NextRecord() = (rr, (000005-001.log: 692), 6022 from previous files, unexpected EOF)
+r.NextRecord() = (rr, (000005-001.log: 692), 6022 from previous files, pebble/record: unexpected EOF)
 
 # Read again, this time pretending we found a third segment with the
 # logNameIndex=002. This helps exercise error conditions switching to a new