Repository: cockroachdb/cockroach
Pull request #146297: changefeed: fix race in cloudstorage sink
Author: login: asg0451
Labels: name: backport-failed
name: backport-all
description: Flags PRs that need to be backported to all supported release branches
name: v25.3.0-prerelease
Created: 2025-05-07T16:53:45Z | Merged: 2025-05-19T21:22:23Z

When using zstd or "fast" gzip compression, the
cloudstorage sink was susceptible to a data race
on its buffered data. This was due to unexpected
asynchronousness in the compression writers.


Fixes: #130656
Fixes: #130651


Release note (bug fix): Fix a data race in the cloudstorage sink.


Diff:
diff --git a/pkg/ccl/changefeedccl/changefeed_test.go b/pkg/ccl/changefeedccl/changefeed_test.go
index 82ba18f425cc..21cf0cdcd0ee 100644
--- a/pkg/ccl/changefeedccl/changefeed_test.go
+++ b/pkg/ccl/changefeedccl/changefeed_test.go
@@ -11573,3 +11573,49 @@ func TestChangefeedMVCCTimestampWithQueries(t *testing.T) {
 
 	cdcTest(t, testFn)
 }
+
+func TestCloudstorageParallelCompression(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	// This test only provides value under race, as it's explicitly testing for
+	// data races between feeds.
+	skip.UnlessUnderRace(t)
+
+	const numFeedsEach = 10
+
+	testutils.RunValues(t, "compression", []string{"zstd", "gzip"}, func(t *testing.T, compression string) {
+		s, cleanup := makeServer(t)
+		defer cleanup()
+
+		sqlDB := sqlutils.MakeSQLRunner(s.DB)
+		sqlDB.Exec(t, `CREATE TABLE foo (a INT);`)
+		sqlDB.Exec(t, `INSERT INTO foo (a) SELECT * FROM generate_series(1, 5000);`)
+
+		t.Logf("inserted into table")
+
+		var jobIDs []int
+		for i := range numFeedsEach {
+			var jobID int
+			sqlDB.QueryRow(t, fmt.Sprintf(`CREATE CHANGEFEED FOR foo INTO 'nodelocal://1/%d-testout' WITH compression='%s', initial_scan='only', format='parquet';`, i, compression)).Scan(&jobID)
+			jobIDs = append(jobIDs, jobID)
+		}
+
+		t.Logf("created changefeeds")
+
+		const duration = 3 * time.Minute
+		const checkStatusInterval = 10 * time.Second
+
+		for start := timeutil.Now(); timeutil.Since(start) < duration; {
+			// Check the statuses of the jobs.
+			for _, jobID := range jobIDs {
+				var status string
+				sqlDB.QueryRow(t, `SELECT status FROM [SHOW JOBS] WHERE job_id = $1`, jobID).Scan(&status)
+				if status != "succeeded" && status != "running" {
+					t.Fatalf("job %d entered unknown state: %s", jobID, status)
+				}
+			}
+			time.Sleep(checkStatusInterval)
+		}
+	})
+}
diff --git a/pkg/ccl/changefeedccl/sink_cloudstorage.go b/pkg/ccl/changefeedccl/sink_cloudstorage.go
index f51b7560d2a6..69c2a9e6b4e7 100644
--- a/pkg/ccl/changefeedccl/sink_cloudstorage.go
+++ b/pkg/ccl/changefeedccl/sink_cloudstorage.go
@@ -66,13 +66,33 @@ func cloudStorageFormatTime(ts hlc.Timestamp) string {
 	return fmt.Sprintf(`%s%09d%010d`, t.Format(f), t.Nanosecond(), ts.Logical)
 }
 
+// byteBufferWithTrackedLength is a bytes.Buffer that also tracks its length
+// separately and atomically. This is useful for codecs that write to the buffer
+// asynchronously, such as pgzip/zstd, because we can't safely call `buf.Len()`
+// while the codec is open, because buf.Write() may be called after
+// codec.Write() returns.
+type byteBufferWithTrackedLength struct {
+	bytes.Buffer
+	len atomic.Int64
+}
+
+func (b *byteBufferWithTrackedLength) Write(p []byte) (n int, err error) {
+	n, err = b.Buffer.Write(p)
+	b.len.Add(int64(n))
+	return n, err
+}
+
+func (b *byteBufferWithTrackedLength) Len() int {
+	return int(b.len.Load())
+}
+
 type cloudStorageSinkFile struct {
 	cloudStorageSinkKey
 	created       time.Time
 	codec         io.WriteCloser
 	rawSize       int
 	numMessages   int
-	buf           bytes.Buffer
+	buf           byteBufferWithTrackedLength
 	alloc         kvevent.Alloc
 	oldestMVCC    hlc.Timestamp
 	parquetCodec  *parquetWriter
diff --git a/pkg/ccl/changefeedccl/sink_cloudstorage_test.go b/pkg/ccl/changefeedccl/sink_cloudstorage_test.go
index a5821b0c89d2..4a0e230abf92 100644
--- a/pkg/ccl/changefeedccl/sink_cloudstorage_test.go
+++ b/pkg/ccl/changefeedccl/sink_cloudstorage_test.go
@@ -37,7 +37,6 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/tabledesc"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
-	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/ioctx"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
@@ -848,7 +847,6 @@ func (o explicitTimestampOracle) inclusiveLowerBoundTS() hlc.Timestamp {
 func TestCloudStorageSinkFastGzip(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
-	skip.UnderRace(t, "#130651")
 
 	ctx := context.Background()
 	settings := cluster.MakeTestingClusterSettings()
diff --git a/pkg/testutils/skip/skip.go b/pkg/testutils/skip/skip.go
index 8c291f14f196..a798d95522a3 100644
--- a/pkg/testutils/skip/skip.go
+++ b/pkg/testutils/skip/skip.go
@@ -86,6 +86,14 @@ func UnderRace(t SkippableTest, args ...interface{}) {
 	}
 }
 
+// UnlessUnderRace skips this test if the race detector is not enabled.
+func UnlessUnderRace(t SkippableTest, args ...interface{}) {
+	t.Helper()
+	if !util.RaceEnabled {
+		maybeSkip(t, "disabled unless under race", args...)
+	}
+}
+
 // UnderRaceWithIssue skips this test if the race detector is enabled,
 // logging the given issue ID as the reason.
 func UnderRaceWithIssue(t SkippableTest, githubIssueID int, args ...interface{}) {


Reviews:
user: login: rharding6373
body: LGTM, nice fix! Only a couple nits
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-05-13T18:51:11Z
user: login: andyyang890
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-13T20:35:08Z
user: login: asg0451
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-16T18:37:33Z
user: login: andyyang890
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-05-19T16:01:48Z
user: login: andyyang890
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-05-19T17:45:32Z

Review comments:
diff_hunk: @@ -102,9 +107,19 @@ var _ io.Writer = &cloudStorageSinkFile{}
 func (f *cloudStorageSinkFile) Write(p []byte) (int, error) {
 	f.rawSize += len(p)
 	if f.codec != nil {
-		return f.codec.Write(p)
+		n, err := f.codec.Write(p) // WRITE
path: pkg/ccl/changefeedccl/sink_cloudstorage.go
user: login: rharding6373
body: nit: left a debugging comment
created_at: 2025-05-13T18:01:12Z
author_association: COLLABORATOR
diff_hunk: @@ -848,7 +847,6 @@ func (o explicitTimestampOracle) inclusiveLowerBoundTS() hlc.Timestamp {
 func TestCloudStorageSinkFastGzip(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
-	skip.UnderRace(t, "#130651")
path: pkg/ccl/changefeedccl/sink_cloudstorage_test.go
user: login: andyyang890
body: nit: add a `Fixes:` comment for this issue
created_at: 2025-05-13T18:23:32Z
author_association: COLLABORATOR
diff_hunk: @@ -586,7 +601,7 @@ func (s *cloudStorageSink) EmitRow(
 	}
 	file.numMessages++
 
-	if int64(file.buf.Len()) > s.targetMaxFileSize {
+	if file.bufLen > s.targetMaxFileSize { // READ
path: pkg/ccl/changefeedccl/sink_cloudstorage.go
user: login: andyyang890
body: nit: leftover comment?
created_at: 2025-05-13T18:26:01Z
author_association: COLLABORATOR
diff_hunk: @@ -68,11 +69,15 @@ func cloudStorageFormatTime(ts hlc.Timestamp) string {
 
 type cloudStorageSinkFile struct {
 	cloudStorageSinkKey
-	created       time.Time
-	codec         io.WriteCloser
-	rawSize       int
-	numMessages   int
-	buf           bytes.Buffer
+	created     time.Time
+	codec       io.WriteCloser
+	rawSize     int
+	numMessages int
+	buf         bytes.Buffer
+	// bufLen tracks the length of buf. Because some of the codecs write to the
+	// buffer asynchronously, we can't safely call `buf.Len()` while the codec
+	// is open.
+	bufLen        int64
path: pkg/ccl/changefeedccl/sink_cloudstorage.go
user: login: andyyang890
body: How awful do you think the performance would be if we wrapped the buffer with a mutex for just the pgzip writer case? I'm a little worried we might forget to update `bufLen` somewhere in the future.
created_at: 2025-05-13T20:32:42Z
author_association: COLLABORATOR
diff_hunk: @@ -11546,3 +11546,43 @@ func TestChangefeedAsSelectForEmptyTable(t *testing.T) {
 
 	cdcTest(t, testFn)
 }
+
+func TestCloudstorageParallelCompression(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	const numFeedsEach = 10
+
+	testutils.RunValues(t, "compression", []string{"zstd", "gzip"}, func(t *testing.T, compression string) {
+		s, cleanup := makeServer(t)
+		defer cleanup()
+
+		sqlDB := sqlutils.MakeSQLRunner(s.DB)
+		sqlDB.Exec(t, `CREATE TABLE foo (a INT);`)
+		sqlDB.Exec(t, `INSERT INTO foo (a) SELECT * FROM generate_series(1, 5000);`)
+
+		t.Logf("inserted into table")
+
+		var jobIDs []int
+		for i := range numFeedsEach {
+			var jobID int
+			sqlDB.QueryRow(t, fmt.Sprintf(`CREATE CHANGEFEED FOR foo INTO 'nodelocal://1/%d-testout' WITH compression='%s', initial_scan='only', format='parquet';`, i, compression)).Scan(&jobID)
+			jobIDs = append(jobIDs, jobID)
+		}
+
+		t.Logf("created changefeeds")
+
+		// Wait for completion.
+		testutils.SucceedsSoon(t, func() error {
path: pkg/ccl/changefeedccl/changefeed_test.go
user: login: asg0451
body: TODO: does this finish now?
created_at: 2025-05-15T16:40:12Z
author_association: CONTRIBUTOR
diff_hunk: @@ -68,11 +69,15 @@ func cloudStorageFormatTime(ts hlc.Timestamp) string {
 
 type cloudStorageSinkFile struct {
 	cloudStorageSinkKey
-	created       time.Time
-	codec         io.WriteCloser
-	rawSize       int
-	numMessages   int
-	buf           bytes.Buffer
+	created     time.Time
+	codec       io.WriteCloser
+	rawSize     int
+	numMessages int
+	buf         bytes.Buffer
+	// bufLen tracks the length of buf. Because some of the codecs write to the
+	// buffer asynchronously, we can't safely call `buf.Len()` while the codec
+	// is open.
+	bufLen        int64
path: pkg/ccl/changefeedccl/sink_cloudstorage.go
user: login: asg0451
body: a mutex wouldnt do the job because we have no way to know the interior state of the gzip writer. i hear your concerns though. let me see if there's a way to make this api a little safer
created_at: 2025-05-15T16:41:11Z
author_association: CONTRIBUTOR
diff_hunk: @@ -848,7 +847,6 @@ func (o explicitTimestampOracle) inclusiveLowerBoundTS() hlc.Timestamp {
 func TestCloudStorageSinkFastGzip(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
-	skip.UnderRace(t, "#130651")
path: pkg/ccl/changefeedccl/sink_cloudstorage_test.go
user: login: andyyang890
body: super nit: also needs to go in the PR desc or it won't close the issue
created_at: 2025-05-19T15:59:48Z
author_association: COLLABORATOR
diff_hunk: @@ -11573,3 +11573,46 @@ func TestChangefeedMVCCTimestampWithQueries(t *testing.T) {
 
 	cdcTest(t, testFn)
 }
+
+func TestCloudstorageParallelCompression(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+	skip.UnlessUnderRace(t)
path: pkg/ccl/changefeedccl/changefeed_test.go
user: login: andyyang890
body: optional super nit: add a newline before this
created_at: 2025-05-19T17:43:26Z
author_association: COLLABORATOR

PR comments:
user: login: cockroach-teamcity
created_at: 2025-05-07T16:53:59Z
body: 

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/cockroachdb/cockroach/146297)
<!-- Reviewable:end -->

author_association: MEMBER
user: login: asg0451
created_at: 2025-05-19T20:18:38Z
body: bors r=andyyang890 
author_association: CONTRIBUTOR
user: login: craig[bot]
created_at: 2025-05-19T21:22:23Z
body: Build succeeded:
  * [lint](https://github.com/cockroachdb/cockroach/actions/runs/15122782519/job/42508752319)
  * [linux_amd64_fips_build](https://github.com/cockroachdb/cockroach/actions/runs/15122782519/job/42508753718)
  * [local_roachtest_fips](https://github.com/cockroachdb/cockroach/actions/runs/15122782519/job/42508753269)
  * [examples_orms](https://github.com/cockroachdb/cockroach/actions/runs/15122782519/job/42508751675)
  * [local_roachtest](https://github.com/cockroachdb/cockroach/actions/runs/15122782519/job/42508752907)
  * [unit_tests](https://github.com/cockroachdb/cockroach/actions/runs/15122782519/job/42508754129)
  * [docker_image_amd64](https://github.com/cockroachdb/cockroach/actions/runs/15122782519/job/42508751962)
  * [acceptance](https://github.com/cockroachdb/cockroach/actions/runs/15122782519/job/42508751292)
  * [check_generated_code](https://github.com/cockroachdb/cockroach/actions/runs/15122782519/job/42508750871)
  * [linux_amd64_build](https://github.com/cockroachdb/cockroach/actions/runs/15122782519/job/42508752607)
author_association: CONTRIBUTOR
user: login: blathers-crl[bot]
created_at: 2025-05-19T21:22:40Z
body: Encountered an error creating backports. Some common things that can go wrong:
1. The backport branch might have already existed.
2. There was a merge conflict.
3. The backport branch contained merge commits.

You might need to create your backport manually using the [backport](https://github.com/benesch/backport) tool.

----

error creating merge commit from 6a48dff2bbd926483c8094f3036a386b8a032600 to blathers/backport-release-24.1-146297: POST https://api.github.com/repos/asg0451/cockroach/merges: 409 Merge conflict []

you may need to manually resolve merge conflicts with the backport tool.

Backport to branch release-24.1 failed. See errors above.

----

error creating merge commit from 6a48dff2bbd926483c8094f3036a386b8a032600 to blathers/backport-release-24.3-146297: POST https://api.github.com/repos/asg0451/cockroach/merges: 409 Merge conflict []

you may need to manually resolve merge conflicts with the backport tool.

Backport to branch release-24.3 failed. See errors above.

----

error creating merge commit from 6a48dff2bbd926483c8094f3036a386b8a032600 to blathers/backport-release-25.1-146297: POST https://api.github.com/repos/asg0451/cockroach/merges: 409 Merge conflict []

you may need to manually resolve merge conflicts with the backport tool.

Backport to branch release-25.1 failed. See errors above.

----

error creating merge commit from 6a48dff2bbd926483c8094f3036a386b8a032600 to blathers/backport-release-25.2-146297: POST https://api.github.com/repos/asg0451/cockroach/merges: 409 Merge conflict []

you may need to manually resolve merge conflicts with the backport tool.

Backport to branch release-25.2 failed. See errors above.

----

<sub>:owl: Hoot! I am a [Blathers](https://github.com/apps/blathers-crl), a bot for [CockroachDB](https://github.com/cockroachdb). My owner is [dev-inf](https://github.com/orgs/cockroachdb/teams/dev-inf).</sub>
author_association: NONE