Repository: cockroachdb/pebble (go)

Issue:
metamorphic: TestMeta failed [compaction cancelled by a concurrent operation]
I think the metamorphic tests need to be adjusted to tolerate background errors.

```
=== NAME  TestMeta/execution/random-005
    meta.go:218: 
        ===== SEED =====
        1695839048661996000
        ===== ERR =====
        exit status 1
        ===== OUT =====
        pebble: compaction cancelled by a concurrent operation, will retry compaction
        
        ===== OPTIONS =====
        [Version]
          pebble_version=0.1
        
        [Options]
          bytes_per_sync=32768
          cache_size=32768
          cleaner=delete
          compaction_debt_concurrency=1073741824
          comparer=pebble.internal.testkeys
          disable_wal=false
          flush_delay_delete_range=975ms
          flush_delay_range_key=25ms
          flush_split_bytes=64
          format_major_version=11
          l0_compaction_concurrency=4
          l0_compaction_file_threshold=2
          l0_compaction_threshold=12
          l0_stop_writes_threshold=71
          lbase_max_bytes=4
          level_multiplier=20
          max_concurrent_compactions=1
          max_manifest_file_size=65536
          max_open_files=1000
          mem_table_size=16777216
          mem_table_stop_writes_threshold=5
          min_deletion_rate=268435456
          merger=pebble.concatenate
          read_compaction_rate=16000
          read_sampling_multiplier=16
          strict_wal_tail=true
          table_cache_shards=10
          table_property_collectors=[]
          validate_on_ingest=false
          wal_dir=
          wal_bytes_per_sync=0
          max_writer_concurrency=0
          force_writer_parallelism=false
          secondary_cache_size_bytes=0
          disable_delete_only_compactions=true
 ```

Pull request description:
metamorphic: Ignore ErrCancelledCompaction in background errors
Previously, if we cancelled a compaction due to an excise or ingest-split, the resultant error would error out the metamorphic test as it wasn't ignored in the metamorphic test's EventListener.

This change updates the ignore rule in the metamorphic test to not exit out on a background error that is ErrCancelledCompaction.

Fixes #2948.

Patch:
diff --git a/compaction.go b/compaction.go
index 530e160b..47abc977 100644
--- a/compaction.go
+++ b/compaction.go
@@ -32,7 +32,10 @@ import (
 )
 
 var errEmptyTable = errors.New("pebble: empty table")
-var errCancelledCompaction = errors.New("pebble: compaction cancelled by a concurrent operation, will retry compaction")
+
+// ErrCancelledCompaction is returned if a compaction is cancelled by a
+// concurrent excise or ingest-split operation.
+var ErrCancelledCompaction = errors.New("pebble: compaction cancelled by a concurrent operation, will retry compaction")
 
 var compactLabels = pprof.Labels("pebble", "compact")
 var flushLabels = pprof.Labels("pebble", "flush")
@@ -2706,7 +2709,7 @@ func (d *DB) compact1(c *compaction, errChannel chan error) (err error) {
 			// the manifest lock, we don't expect this bool to change its value
 			// as only the holder of the manifest lock will ever write to it.
 			if c.cancel.Load() {
-				err = firstError(err, errCancelledCompaction)
+				err = firstError(err, ErrCancelledCompaction)
 			}
 			if err != nil {
 				// logAndApply calls logUnlock. If we didn't call it, we need to call
@@ -2871,7 +2874,7 @@ func (d *DB) runCompaction(
 	}
 
 	if c.cancel.Load() {
-		return ve, nil, stats, errCancelledCompaction
+		return ve, nil, stats, ErrCancelledCompaction
 	}
 
 	// Release the d.mu lock while doing I/O.
@@ -3004,7 +3007,7 @@ func (d *DB) runCompaction(
 	newOutput := func() error {
 		// Check if we've been cancelled by a concurrent operation.
 		if c.cancel.Load() {
-			return errCancelledCompaction
+			return ErrCancelledCompaction
 		}
 		fileMeta := &fileMetadata{}
 		d.mu.Lock()


Test patch:
diff --git a/metamorphic/test.go b/metamorphic/test.go
index f1ec3794..90d2d27b 100644
--- a/metamorphic/test.go
+++ b/metamorphic/test.go
@@ -73,7 +73,7 @@ func (t *test) init(h *history, dir string, testOpts *TestOptions) error {
 	// difference between in-memory and on-disk which causes different code paths
 	// and timings to be exercised.
 	maybeExit := func(err error) {
-		if err == nil || errors.Is(err, errorfs.ErrInjected) {
+		if err == nil || errors.Is(err, errorfs.ErrInjected) || errors.Is(err, pebble.ErrCancelledCompaction) {
 			return
 		}
 		t.maybeSaveData()