Repository: cockroachdb/cockroach
Pull request #143991: changefeedccl: add roachtest and parallel test for enriched envelope
Author: login: asg0451
Labels: name: v25.2.0-prerelease
Created: 2025-04-07T17:17:47Z | Merged: 2025-04-08T20:44:17Z

Add a roachtest and a parallelized test for enriched envelope feeds.

Part of: #139660

Release note: None


Diff:
diff --git a/pkg/ccl/changefeedccl/cdctest/validator.go b/pkg/ccl/changefeedccl/cdctest/validator.go
index dca57702f43c..d47c0f9bc608 100644
--- a/pkg/ccl/changefeedccl/cdctest/validator.go
+++ b/pkg/ccl/changefeedccl/cdctest/validator.go
@@ -1017,21 +1017,26 @@ func ParseJSONValueTimestamps(v []byte) (updated, resolved hlc.Timestamp, err er
 	var valueRaw struct {
 		Resolved string `json:"resolved"`
 		Updated  string `json:"updated"`
+		Source   struct {
+			TsHLC string `json:"ts_hlc"`
+		} `json:"source"`
 	}
 	if err := gojson.Unmarshal(v, &valueRaw); err != nil {
 		return hlc.Timestamp{}, hlc.Timestamp{}, errors.Wrapf(err, "parsing [%s] as json", v)
 	}
+
 	if valueRaw.Updated != `` {
-		var err error
-		updated, err = hlc.ParseHLC(valueRaw.Updated)
-		if err != nil {
+		if updated, err = hlc.ParseHLC(valueRaw.Updated); err != nil {
 			return hlc.Timestamp{}, hlc.Timestamp{}, err
 		}
 	}
 	if valueRaw.Resolved != `` {
-		var err error
-		resolved, err = hlc.ParseHLC(valueRaw.Resolved)
-		if err != nil {
+		if resolved, err = hlc.ParseHLC(valueRaw.Resolved); err != nil {
+			return hlc.Timestamp{}, hlc.Timestamp{}, err
+		}
+	}
+	if valueRaw.Source.TsHLC != `` {
+		if updated, err = hlc.ParseHLC(valueRaw.Source.TsHLC); err != nil {
 			return hlc.Timestamp{}, hlc.Timestamp{}, err
 		}
 	}
diff --git a/pkg/ccl/changefeedccl/changefeed_test.go b/pkg/ccl/changefeedccl/changefeed_test.go
index bc9a47e11b8b..ef91938d56ba 100644
--- a/pkg/ccl/changefeedccl/changefeed_test.go
+++ b/pkg/ccl/changefeedccl/changefeed_test.go
@@ -4193,6 +4193,99 @@ func TestChangefeedEnriched(t *testing.T) {
 	}
 }
 
+// TestChangefeedsParallelEnriched tests that multiple changefeeds can run in
+// parallel with the enriched envelope. It is most useful under race, to ensure
+// that there is no accidental data races in the encoders and source providers.
+func TestChangefeedsParallelEnriched(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+	ctx := context.Background()
+
+	const numFeeds = 10
+	const maxIterations = 1_000_000_000
+	const maxRows = 100
+
+	testFn := func(t *testing.T, s TestServer, f cdctest.TestFeedFactory) {
+		db := sqlutils.MakeSQLRunner(s.DB)
+		db.Exec(t, `CREATE TABLE foo (a INT PRIMARY KEY, b STRING)`)
+
+		ctx, cancel := context.WithCancel(ctx)
+
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			db := sqlutils.MakeSQLRunner(s.Server.SQLConn(t))
+			var i int
+			for i = 0; i < maxIterations && ctx.Err() == nil; i++ {
+				db.Exec(t, `UPSERT INTO d.foo VALUES ($1, $2)`, i%maxRows, fmt.Sprintf("hello %d", i))
+			}
+		}()
+
+		opts := `envelope='enriched'`
+
+		_, isKafka := f.(*kafkaFeedFactory)
+		useAvro := isKafka && rand.Intn(2) == 0
+		if useAvro {
+			t.Logf("using avro")
+			opts += `, format='avro'`
+		}
+		var feeds []cdctest.TestFeed
+		for range numFeeds {
+			feed := feed(t, f, fmt.Sprintf(`CREATE CHANGEFEED FOR foo WITH %s`, opts))
+			feeds = append(feeds, feed)
+		}
+
+		// consume from the feeds
+		for _, feed := range feeds {
+			feed := feed
+			msgCount := 0
+
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				for ctx.Err() == nil {
+					_, err := feed.Next()
+					if err != nil {
+						if errors.Is(err, context.Canceled) {
+							t.Errorf("error reading from feed: %v", err)
+						}
+						break
+					}
+					msgCount++
+				}
+				assert.GreaterOrEqual(t, msgCount, 0)
+			}()
+		}
+
+		// let the feeds run for a few seconds
+		select {
+		case <-time.After(5 * time.Second):
+		case <-ctx.Done():
+			t.Fatalf("%v", ctx.Err())
+		}
+
+		cancel()
+
+		for _, feed := range feeds {
+			closeFeed(t, feed)
+		}
+
+		doneWaiting := make(chan struct{})
+		go func() {
+			defer close(doneWaiting)
+			wg.Wait()
+		}()
+		select {
+		case <-doneWaiting:
+		case <-time.After(5 * time.Second):
+			t.Fatalf("timed out waiting for goroutines to finish")
+		}
+	}
+	// Sinkless testfeeds have some weird shutdown behaviours, so exclude them for now.
+	cdcTest(t, testFn, feedTestRestrictSinks("kafka", "pubsub", "webhook"))
+}
+
 func TestChangefeedEnrichedAvro(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
diff --git a/pkg/cmd/roachtest/tests/cdc.go b/pkg/cmd/roachtest/tests/cdc.go
index 00fbaca8a036..fafcb6902ed3 100644
--- a/pkg/cmd/roachtest/tests/cdc.go
+++ b/pkg/cmd/roachtest/tests/cdc.go
@@ -517,6 +517,7 @@ func makeDefaultFeatureFlags() cdcFeatureFlags {
 type feedArgs struct {
 	sinkType        sinkType
 	targets         []string
+	envelope        string
 	opts            map[string]string
 	assumeRole      string
 	tolerateErrors  bool
@@ -547,15 +548,15 @@ func (ct *cdcTester) newChangefeed(args feedArgs) changefeedJob {
 
 	targetsStr := strings.Join(args.targets, ", ")
 
+	if args.envelope == "" {
+		args.envelope = "wrapped"
+	}
+
 	feedOptions := make(map[string]string)
 	feedOptions["min_checkpoint_frequency"] = "'10s'"
+	feedOptions["envelope"] = args.envelope
 	if args.sinkType == cloudStorageSink || args.sinkType == webhookSink {
-		// Webhook and cloudstorage don't have a concept of keys and therefore
-		// require envelope=wrapped
-		feedOptions["envelope"] = "wrapped"
-
 		feedOptions["resolved"] = "'10s'"
-
 	} else {
 		feedOptions["resolved"] = ""
 	}
@@ -2311,6 +2312,40 @@ func registerCDC(r registry.Registry) {
 		Timeout:          1 * time.Hour,
 		Run:              runCDCMultipleSchemaChanges,
 	})
+	r.Add(registry.TestSpec{
+		Name:             "cdc/tpcc-100/10min/sink=kafka/envelope=enriched",
+		Owner:            registry.OwnerCDC,
+		Benchmark:        true,
+		Cluster:          r.MakeClusterSpec(4, spec.WorkloadNode(), spec.CPU(16)),
+		Leases:           registry.MetamorphicLeases,
+		CompatibleClouds: registry.AllClouds,
+		Suites:           registry.Suites(registry.Nightly),
+		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
+			ct := newCDCTester(ctx, t, c)
+			defer ct.Close()
+
+			ct.runTPCCWorkload(tpccArgs{warehouses: 100, duration: "10m"})
+
+			feed := ct.newChangefeed(feedArgs{
+				sinkType: kafkaSink,
+				envelope: "enriched",
+				targets:  allTpccTargets,
+				kafkaArgs: kafkaFeedArgs{
+					validateOrder: true,
+				},
+				opts: map[string]string{
+					"initial_scan":        "'no'",
+					"updated":             "",
+					"enriched_properties": "source",
+				},
+			})
+			ct.runFeedLatencyVerifier(feed, latencyTargets{
+				initialScanLatency: 3 * time.Minute,
+				steadyLatency:      10 * time.Minute,
+			})
+			ct.waitForWorkload()
+		},
+	})
 }
 
 const (


Reviews:
user: login: andyyang890
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-08T19:19:47Z
user: login: asg0451
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-08T19:37:50Z

Review comments:
diff_hunk: @@ -1017,21 +1017,26 @@ func ParseJSONValueTimestamps(v []byte) (updated, resolved hlc.Timestamp, err er
 	var valueRaw struct {
 		Resolved string `json:"resolved"`
 		Updated  string `json:"updated"`
+		Source   struct {
+			TsHLC string `json:"ts_hlc"`
+		} `json:"source"`
 	}
 	if err := gojson.Unmarshal(v, &valueRaw); err != nil {
 		return hlc.Timestamp{}, hlc.Timestamp{}, errors.Wrapf(err, "parsing [%s] as json", v)
 	}
+
 	if valueRaw.Updated != `` {
-		var err error
-		updated, err = hlc.ParseHLC(valueRaw.Updated)
-		if err != nil {
+		if updated, err = hlc.ParseHLC(valueRaw.Updated); err != nil {
 			return hlc.Timestamp{}, hlc.Timestamp{}, err
 		}
 	}
 	if valueRaw.Resolved != `` {
-		var err error
-		resolved, err = hlc.ParseHLC(valueRaw.Resolved)
-		if err != nil {
+		if resolved, err = hlc.ParseHLC(valueRaw.Resolved); err != nil {
+			return hlc.Timestamp{}, hlc.Timestamp{}, err
+		}
+	}
+	if valueRaw.Source.TsHLC != `` {
path: pkg/ccl/changefeedccl/cdctest/validator.go
user: login: andyyang890
body: Should we have an assertion that `valueRaw.Updated` and `valueRaw.Source.TsHLC` can't both be set at the same time?
created_at: 2025-04-08T18:50:57Z
author_association: COLLABORATOR
diff_hunk: @@ -829,7 +829,7 @@ func TestChangefeedDiff(t *testing.T) {
 		})
 	}
 
-	cdcTest(t, testFn)
+	cdcTest(t, testFn, feedTestForceSink("kafka"))
path: pkg/ccl/changefeedccl/changefeed_test.go
user: login: andyyang890
body: Is this change intentional? If so, why?
created_at: 2025-04-08T18:51:55Z
author_association: COLLABORATOR
diff_hunk: @@ -1479,7 +1480,7 @@ func registerCDC(r registry.Registry) {
 			ct := newCDCTester(ctx, t, c)
 			defer ct.Close()
 
-			ct.runTPCCWorkload(tpccArgs{warehouses: 1000, duration: "120m"})
+			ct.runTPCCWorkload(tpccArgs{warehouses: 10, duration: "10m"})
path: pkg/cmd/roachtest/tests/cdc.go
user: login: andyyang890
body: Is this a leftover debug change?
created_at: 2025-04-08T18:53:11Z
author_association: COLLABORATOR
diff_hunk: @@ -2311,6 +2312,41 @@ func registerCDC(r registry.Registry) {
 		Timeout:          1 * time.Hour,
 		Run:              runCDCMultipleSchemaChanges,
 	})
+
path: pkg/cmd/roachtest/tests/cdc.go
user: login: andyyang890
body: nit: delete this newline
created_at: 2025-04-08T18:54:28Z
author_association: COLLABORATOR
diff_hunk: @@ -4193,6 +4193,98 @@ func TestChangefeedEnriched(t *testing.T) {
 	}
 }
 
+// TestChangefeedsParallelEnriched tests that multiple changefeeds can run
+// in parallel with the enriched envelope. It is most useful under race.
+func TestChangefeedsParallelEnriched(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+	ctx := context.Background()
+
+	const count = 10
path: pkg/ccl/changefeedccl/changefeed_test.go
user: login: andyyang890
body: nit: s/count/numChangefeeds
created_at: 2025-04-08T18:58:05Z
author_association: COLLABORATOR
diff_hunk: @@ -4193,6 +4193,98 @@ func TestChangefeedEnriched(t *testing.T) {
 	}
 }
 
+// TestChangefeedsParallelEnriched tests that multiple changefeeds can run
+// in parallel with the enriched envelope. It is most useful under race.
+func TestChangefeedsParallelEnriched(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+	ctx := context.Background()
+
+	const count = 10
+	const maxIterations = 1_000_000_000
+	const maxRows = 100
+
+	testFn := func(t *testing.T, s TestServer, f cdctest.TestFeedFactory) {
+		db := sqlutils.MakeSQLRunner(s.DB)
+		db.Exec(t, `CREATE TABLE foo (a INT PRIMARY KEY, b STRING)`)
+
+		ctx, cancel := context.WithCancel(ctx)
+
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			db := sqlutils.MakeSQLRunner(s.Server.SQLConn(t))
+			var i int
+			for i = 0; i < maxIterations && ctx.Err() == nil; i++ {
+				db.Exec(t, `UPSERT INTO d.foo VALUES ($1, $2)`, i%maxRows, fmt.Sprintf("hello %d", i))
+			}
+		}()
+
+		opts := `envelope='enriched'`
+
+		_, isKafka := f.(*kafkaFeedFactory)
+		useAvro := isKafka && rand.Intn(2) == 0
+		if useAvro {
+			t.Logf("using avro")
+			opts += `, format='avro'`
+		}
+		var feeds []cdctest.TestFeed
+		for range count {
+			feed := feed(t, f, fmt.Sprintf(`CREATE CHANGEFEED FOR foo WITH %s`, opts))
+			feeds = append(feeds, feed)
+		}
+
+		// consume from the feeds
+		for _, feed := range feeds {
+			feed := feed
+			msgCount := 0
+
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				for ctx.Err() == nil {
+					_, err := feed.Next()
+					if err != nil {
+						if errors.Is(err, context.Canceled) {
+							t.Errorf("error reading from feed: %v", err)
+						}
+						break
+					}
+					msgCount++
+				}
+				assert.GreaterOrEqual(t, msgCount, 0)
+			}()
+		}
+
+		// let the feeds run for a few seconds
+		select {
+		case <-time.After(5 * time.Second):
+		case <-ctx.Done():
+			t.Fatalf("context cancelled: %v", ctx.Err())
path: pkg/ccl/changefeedccl/changefeed_test.go
user: login: andyyang890
body: nit: the error will already say `context canceled` right
created_at: 2025-04-08T19:16:39Z
author_association: COLLABORATOR
diff_hunk: @@ -4193,6 +4193,98 @@ func TestChangefeedEnriched(t *testing.T) {
 	}
 }
 
+// TestChangefeedsParallelEnriched tests that multiple changefeeds can run
path: pkg/ccl/changefeedccl/changefeed_test.go
user: login: andyyang890
body: Do you mind adding a comment explaining the motivation for this test? Is there some reason this wouldn't work (e.g. is there some kind of shared state for enriched changefeeds)?
created_at: 2025-04-08T19:19:18Z
author_association: COLLABORATOR
diff_hunk: @@ -1017,21 +1017,26 @@ func ParseJSONValueTimestamps(v []byte) (updated, resolved hlc.Timestamp, err er
 	var valueRaw struct {
 		Resolved string `json:"resolved"`
 		Updated  string `json:"updated"`
+		Source   struct {
+			TsHLC string `json:"ts_hlc"`
+		} `json:"source"`
 	}
 	if err := gojson.Unmarshal(v, &valueRaw); err != nil {
 		return hlc.Timestamp{}, hlc.Timestamp{}, errors.Wrapf(err, "parsing [%s] as json", v)
 	}
+
 	if valueRaw.Updated != `` {
-		var err error
-		updated, err = hlc.ParseHLC(valueRaw.Updated)
-		if err != nil {
+		if updated, err = hlc.ParseHLC(valueRaw.Updated); err != nil {
 			return hlc.Timestamp{}, hlc.Timestamp{}, err
 		}
 	}
 	if valueRaw.Resolved != `` {
-		var err error
-		resolved, err = hlc.ParseHLC(valueRaw.Resolved)
-		if err != nil {
+		if resolved, err = hlc.ParseHLC(valueRaw.Resolved); err != nil {
+			return hlc.Timestamp{}, hlc.Timestamp{}, err
+		}
+	}
+	if valueRaw.Source.TsHLC != `` {
path: pkg/ccl/changefeedccl/cdctest/validator.go
user: login: asg0451
body: Don't think we have to. We don't have that for Updated and Resolved.
created_at: 2025-04-08T19:37:50Z
author_association: CONTRIBUTOR

PR comments:
user: login: cockroach-teamcity
created_at: 2025-04-07T17:17:57Z
body: 

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/cockroachdb/cockroach/143991)
<!-- Reviewable:end -->

author_association: MEMBER
user: login: asg0451
created_at: 2025-04-08T19:40:11Z
body: tftr @andyyang890 

author_association: CONTRIBUTOR
user: login: asg0451
created_at: 2025-04-08T19:40:15Z
body: bors r=andyyang890 
author_association: CONTRIBUTOR
user: login: craig[bot]
created_at: 2025-04-08T20:44:17Z
body: Build succeeded:
  * [local_roachtest](https://github.com/cockroachdb/cockroach/actions/runs/14342656410/job/40205524437)
  * [check_generated_code](https://github.com/cockroachdb/cockroach/actions/runs/14342656410/job/40205521573)
  * [acceptance](https://github.com/cockroachdb/cockroach/actions/runs/14342656410/job/40205521028)
  * [docker_image_amd64](https://github.com/cockroachdb/cockroach/actions/runs/14342656410/job/40205522078)
  * [local_roachtest_fips](https://github.com/cockroachdb/cockroach/actions/runs/14342656410/job/40205524807)
  * [examples_orms](https://github.com/cockroachdb/cockroach/actions/runs/14342656410/job/40205522562)
  * [linux_amd64_build](https://github.com/cockroachdb/cockroach/actions/runs/14342656410/job/40205523641)
  * unit_tests
  * [linux_amd64_fips_build](https://github.com/cockroachdb/cockroach/actions/runs/14342656410/job/40205524024)
  * [lint](https://github.com/cockroachdb/cockroach/actions/runs/14342656410/job/40205523055)
author_association: CONTRIBUTOR