Repository: cockroachdb/cockroach
Pull request #143069: cloud/amazon: add option to skip checksums
Author: login: dt
Labels: name: backport-24.3.x
description: Flags PRs that need to be backported to 24.3
name: v25.2.0-prerelease
Created: 2025-03-18T15:15:27Z | Merged: 2025-03-31T18:07:03Z

Our files (SSTs) often have internal checksums anyway, and we have ecplicit checksum files for those that don't that are cloud agnostic, so we do not generally need the extra checksums in s3-specific metadata. Computing them has non-zero cost as well, so users may wish to disable them. Indeed, we almost did so by default a couple years ago before realizing that they are required by the S3 API if certain other features, like object locking, are enabled. So while some users may not be able to or want to disable them, e.g. if they're using object locking, we can provide the option to do so to those who would want it. This may also be useful to users of certain s3-like storage services that have different compatibility with different checksums.

Release note: none.
Epic: none.


Diff:
diff --git a/pkg/cloud/amazon/s3_storage.go b/pkg/cloud/amazon/s3_storage.go
index 82eb1f231fbd..cfe6fdf16cc8 100644
--- a/pkg/cloud/amazon/s3_storage.go
+++ b/pkg/cloud/amazon/s3_storage.go
@@ -57,6 +57,8 @@ const (
 	AWSEndpointParam = "AWS_ENDPOINT"
 	// AWSEndpointParam is the query parameter for UsePathStyle in S3 options.
 	AWSUsePathStyle = "AWS_USE_PATH_STYLE"
+	// AWSSkipChecksumParam is the query parameter for SkipChecksum in S3 options.
+	AWSSkipChecksumParam = "AWS_SKIP_CHECKSUM"
 
 	// AWSServerSideEncryptionMode is the query parameter in an AWS URI, for the
 	// mode to be used for server side encryption. It can either be AES256 or
@@ -200,6 +202,7 @@ type s3ClientConfig struct {
 	assumeRoleProvider                                           roleProvider
 	delegateRoleProviders                                        []roleProvider
 
+	skipChecksum bool
 	// log.V(2) decides session init params so include it in key.
 	verbose bool
 }
@@ -232,6 +235,7 @@ func clientConfig(conf *cloudpb.ExternalStorage_S3) s3ClientConfig {
 	return s3ClientConfig{
 		endpoint:              conf.Endpoint,
 		usePathStyle:          conf.UsePathStyle,
+		skipChecksum:          conf.SkipChecksum,
 		region:                conf.Region,
 		bucket:                conf.Bucket,
 		accessKey:             conf.AccessKey,
@@ -280,6 +284,9 @@ func S3URI(bucket, path string, conf *cloudpb.ExternalStorage_S3) string {
 	if conf.UsePathStyle {
 		q.Set(AWSUsePathStyle, "true")
 	}
+	if conf.SkipChecksum {
+		q.Set(AWSSkipChecksumParam, "true")
+	}
 	if conf.AssumeRoleProvider.Role != "" {
 		roleProviderStrings := make([]string, 0, len(conf.DelegateRoleProviders)+1)
 		for _, p := range conf.DelegateRoleProviders {
@@ -325,6 +332,15 @@ func parseS3URL(uri *url.URL) (cloudpb.ExternalStorage, error) {
 			return cloudpb.ExternalStorage{}, errors.Wrapf(err, "cannot parse %s as bool", AWSUsePathStyle)
 		}
 	}
+	skipChecksumStr := s3URL.ConsumeParam(AWSSkipChecksumParam)
+	skipChecksumBool := false
+	if skipChecksumStr != "" {
+		var err error
+		skipChecksumBool, err = strconv.ParseBool(skipChecksumStr)
+		if err != nil {
+			return cloudpb.ExternalStorage{}, errors.Wrapf(err, "cannot parse %s as bool", AWSSkipChecksumParam)
+		}
+	}
 
 	conf.S3Config = &cloudpb.ExternalStorage_S3{
 		Bucket:                s3URL.Host,
@@ -334,6 +350,7 @@ func parseS3URL(uri *url.URL) (cloudpb.ExternalStorage, error) {
 		TempToken:             s3URL.ConsumeParam(AWSTempTokenParam),
 		Endpoint:              s3URL.ConsumeParam(AWSEndpointParam),
 		UsePathStyle:          pathStyleBool,
+		SkipChecksum:          skipChecksumBool,
 		Region:                s3URL.ConsumeParam(S3RegionParam),
 		Auth:                  s3URL.ConsumeParam(cloud.AuthParam),
 		ServerEncMode:         s3URL.ConsumeParam(AWSServerSideEncryptionMode),
@@ -590,6 +607,11 @@ func (s *s3Storage) newClient(ctx context.Context) (s3Client, string, error) {
 		return s3Client{}, "", errors.Wrap(err, "could not initialize an aws config")
 	}
 
+	if s.opts.skipChecksum {
+		cfg.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
+		cfg.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
+	}
+
 	var endpointURI string
 	if s.opts.endpoint != "" {
 		var err error
@@ -728,7 +750,7 @@ func (s *s3Storage) putUploader(ctx context.Context, basename string) (io.WriteC
 
 	buf := bytes.NewBuffer(make([]byte, 0, 4<<20))
 
-	return &putUploader{
+	uploader := &putUploader{
 		b: buf,
 		input: &s3.PutObjectInput{
 			Bucket:               s.bucket,
@@ -739,7 +761,11 @@ func (s *s3Storage) putUploader(ctx context.Context, basename string) (io.WriteC
 			ChecksumAlgorithm:    checksumAlgorithm,
 		},
 		client: client,
-	}, nil
+	}
+	if s.conf.SkipChecksum {
+		uploader.input.ChecksumAlgorithm = ""
+	}
+	return uploader, nil
 }
 
 func (s *s3Storage) Writer(ctx context.Context, basename string) (io.WriteCloser, error) {
@@ -757,8 +783,7 @@ func (s *s3Storage) Writer(ctx context.Context, basename string) (io.WriteCloser
 	return cloud.BackgroundPipe(ctx, func(ctx context.Context, r io.Reader) error {
 		defer sp.Finish()
 		// Upload the file to S3.
-		// TODO(dt): test and tune the uploader parameters.
-		_, err := uploader.Upload(ctx, &s3.PutObjectInput{
+		input := &s3.PutObjectInput{
 			Bucket:               s.bucket,
 			Key:                  aws.String(path.Join(s.prefix, basename)),
 			Body:                 r,
@@ -766,7 +791,13 @@ func (s *s3Storage) Writer(ctx context.Context, basename string) (io.WriteCloser
 			SSEKMSKeyId:          nilIfEmpty(s.conf.ServerKMSID),
 			StorageClass:         types.StorageClass(s.conf.StorageClass),
 			ChecksumAlgorithm:    checksumAlgorithm,
-		})
+		}
+
+		if s.conf.SkipChecksum {
+			input.ChecksumAlgorithm = ""
+		}
+
+		_, err := uploader.Upload(ctx, input)
 		err = interpretAWSError(err)
 		err = errors.Wrap(err, "upload failed")
 		// Mark with ctx's error for upstream code to not interpret this as
@@ -787,7 +818,9 @@ func (s *s3Storage) openStreamAt(
 	if err != nil {
 		return nil, err
 	}
-	req := &s3.GetObjectInput{Bucket: s.bucket, Key: aws.String(path.Join(s.prefix, basename))}
+	req := &s3.GetObjectInput{
+		Bucket: s.bucket, Key: aws.String(path.Join(s.prefix, basename)),
+	}
 	if endPos != 0 {
 		if pos >= endPos {
 			return nil, io.EOF
diff --git a/pkg/cloud/amazon/s3_storage_test.go b/pkg/cloud/amazon/s3_storage_test.go
index 9f0f8ad52191..f439950a67ec 100644
--- a/pkg/cloud/amazon/s3_storage_test.go
+++ b/pkg/cloud/amazon/s3_storage_test.go
@@ -167,6 +167,22 @@ func TestPutS3(t *testing.T) {
 				cloudtestutils.CheckExportStore(t, info)
 			})
 
+			t.Run("skip-checksums", func(t *testing.T) {
+				if locked {
+					skip.IgnoreLint(t, "object-locked buckets do not support skipping checksums")
+				}
+				skipIfNoDefaultConfig(t, ctx)
+				info := cloudtestutils.StoreInfo{
+					URI: fmt.Sprintf(
+						"s3://%s/%s-%d?%s=%s&%s=true",
+						bucket, "backup-test-skip-checksums", testID,
+						cloud.AuthParam, cloud.AuthParamImplicit, AWSSkipChecksumParam,
+					),
+					User: user,
+				}
+				cloudtestutils.CheckExportStore(t, info)
+			})
+
 			t.Run("server-side-encryption-invalid-params", func(t *testing.T) {
 				skipIfNoDefaultConfig(t, ctx)
 				// Unsupported server side encryption option.
@@ -357,6 +373,7 @@ func TestPutS3Endpoint(t *testing.T) {
 		}
 		cloudtestutils.CheckExportStore(t, info)
 	})
+
 	t.Run("use-path-style", func(t *testing.T) {
 		// EngFlow machines have no internet access, and queries even to localhost will time out.
 		// So this test is skipped above.
diff --git a/pkg/cloud/cloudpb/external_storage.proto b/pkg/cloud/cloudpb/external_storage.proto
index 43ace0c6661c..65e890de6a14 100644
--- a/pkg/cloud/cloudpb/external_storage.proto
+++ b/pkg/cloud/cloudpb/external_storage.proto
@@ -63,6 +63,7 @@ message ExternalStorage {
     string temp_token = 5;
     string endpoint = 6;
     bool use_path_style=16;
+    bool skip_checksum=17;
     string region = 7;
     string auth = 8;
     string server_enc_mode  = 9;
@@ -91,7 +92,10 @@ message ExternalStorage {
     // role chain. These roles will be assumed in the order they appear in the
     // list so that the role specified in AssumeRoleProvider can be assumed.
     repeated AssumeRoleProvider delegate_role_providers = 15 [(gogoproto.nullable) = false];
+
+    // Next ID: 18;
   }
+  
   message GCS {
     string bucket = 1;
     string prefix = 2;


Reviews:
user: login: benbardin
body: Add a unit test? Also note somewhere that this isn't compatible with object locking. Probably in-code for us, and in the Release note so Kathryn can call that out. Otherwise LGTM, thank you!
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-18T15:42:04Z
user: login: msbutler
body: If it is easy to add a test case to the cloud unit test suite, that would be great.
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-03-19T12:14:41Z
user: login: msbutler
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-03-28T16:31:26Z
user: login: dt
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-28T16:37:22Z
user: login: dt
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-28T16:41:06Z
user: login: msbutler
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-28T16:53:36Z
user: login: dt
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-28T20:35:03Z

Review comments:
diff_hunk: @@ -57,6 +57,8 @@ const (
 	AWSEndpointParam = "AWS_ENDPOINT"
 	// AWSEndpointParam is the query parameter for UsePathStyle in S3 options.
 	AWSUsePathStyle = "AWS_USE_PATH_STYLE"
+	// AWSSkipChecksumParam is the query parameter for SkipChecksum in S3 options.
path: pkg/cloud/amazon/s3_storage.go
user: login: msbutler
body: my inclination is to leave this option undocumented for now, so should we remove the release note?
created_at: 2025-03-28T16:27:00Z
author_association: COLLABORATOR
diff_hunk: @@ -57,6 +57,8 @@ const (
 	AWSEndpointParam = "AWS_ENDPOINT"
 	// AWSEndpointParam is the query parameter for UsePathStyle in S3 options.
 	AWSUsePathStyle = "AWS_USE_PATH_STYLE"
+	// AWSSkipChecksumParam is the query parameter for SkipChecksum in S3 options.
path: pkg/cloud/amazon/s3_storage.go
user: login: msbutler
body: or maybe the release note should say that this option is only for s3 look a likes that do not support s3's checksum implementation? Given how bespoke the fix is, i would not be surprised if this break next time we update the sdk.
created_at: 2025-03-28T16:31:07Z
author_association: COLLABORATOR
diff_hunk: @@ -57,6 +57,8 @@ const (
 	AWSEndpointParam = "AWS_ENDPOINT"
 	// AWSEndpointParam is the query parameter for UsePathStyle in S3 options.
 	AWSUsePathStyle = "AWS_USE_PATH_STYLE"
+	// AWSSkipChecksumParam is the query parameter for SkipChecksum in S3 options.
path: pkg/cloud/amazon/s3_storage.go
user: login: dt
body: I don't read the release note as promising anything is now supported/won't break so I don't mind it as is, just stating that there is this option and it does this narrowly defined thing. I wouldn't want to make a stronger statement, like "with this option we support $s3-like", without automated testing of that statement that would indeed catch breakages of that support by future upgrades, but I don't think the current note is making a claim like that.

I can remove the note entirely if you'd prefer, but IMO it doesn't feel like an excessive liability as-is.
created_at: 2025-03-28T16:37:22Z
author_association: CONTRIBUTOR
diff_hunk: @@ -57,6 +57,8 @@ const (
 	AWSEndpointParam = "AWS_ENDPOINT"
 	// AWSEndpointParam is the query parameter for UsePathStyle in S3 options.
 	AWSUsePathStyle = "AWS_USE_PATH_STYLE"
+	// AWSSkipChecksumParam is the query parameter for SkipChecksum in S3 options.
path: pkg/cloud/amazon/s3_storage.go
user: login: dt
body: > say that this option is only for s3 look a likes 

I'd rather not do this one: _this_ feels like it raises more questions than it answers or that I really _want_ to answer for -- I'd rather keep the note, for now, strictly factual and let inferences on what it means to any particular user be an exercise for, and responsibility of, the reader.
created_at: 2025-03-28T16:41:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -57,6 +57,8 @@ const (
 	AWSEndpointParam = "AWS_ENDPOINT"
 	// AWSEndpointParam is the query parameter for UsePathStyle in S3 options.
 	AWSUsePathStyle = "AWS_USE_PATH_STYLE"
+	// AWSSkipChecksumParam is the query parameter for SkipChecksum in S3 options.
path: pkg/cloud/amazon/s3_storage.go
user: login: msbutler
body: i think i'd like to leave this option undocumented for now.
created_at: 2025-03-28T16:53:36Z
author_association: COLLABORATOR
diff_hunk: @@ -57,6 +57,8 @@ const (
 	AWSEndpointParam = "AWS_ENDPOINT"
 	// AWSEndpointParam is the query parameter for UsePathStyle in S3 options.
 	AWSUsePathStyle = "AWS_USE_PATH_STYLE"
+	// AWSSkipChecksumParam is the query parameter for SkipChecksum in S3 options.
path: pkg/cloud/amazon/s3_storage.go
user: login: dt
body: okay, I'll remove the release note entirely and let you work with @alicia-l2 and @katmayb work out if/when/what we say in docs.
created_at: 2025-03-28T20:35:02Z
author_association: CONTRIBUTOR

PR comments:
user: login: blathers-crl[bot]
created_at: 2025-03-18T15:15:31Z
body: It looks like your PR touches production code but doesn't add or edit any test code. Did you consider adding tests to your PR?

<sub>:owl: Hoot! I am a [Blathers](https://github.com/apps/blathers-crl), a bot for [CockroachDB](https://github.com/cockroachdb). My owner is [dev-inf](https://github.com/orgs/cockroachdb/teams/dev-inf).</sub>
author_association: NONE
user: login: cockroach-teamcity
created_at: 2025-03-18T15:15:39Z
body: 

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/cockroachdb/cockroach/143069)
<!-- Reviewable:end -->

author_association: MEMBER
user: login: dt
created_at: 2025-03-28T15:40:14Z
body: Added another knob that seems to change how GET requests are sent as well.

And unit test added and cloud unit tests are green: https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_CloudUnitTests/19249049?buildTab=tests&name=puts3%2Fregular-bucket%2Fskip
author_association: CONTRIBUTOR
user: login: dt
created_at: 2025-03-31T17:14:38Z
body: TFTR!

bors r+
author_association: CONTRIBUTOR
user: login: craig[bot]
created_at: 2025-03-31T18:07:02Z
body: Build succeeded:
  * [examples_orms](https://github.com/cockroachdb/cockroach/actions/runs/14177411880/job/39715532285)
  * [local_roachtest](https://github.com/cockroachdb/cockroach/actions/runs/14177411880/job/39715534360)
  * [check_generated_code](https://github.com/cockroachdb/cockroach/actions/runs/14177411880/job/39715532729)
  * [linux_amd64_fips_build](https://github.com/cockroachdb/cockroach/actions/runs/14177411880/job/39715535277)
  * [lint](https://github.com/cockroachdb/cockroach/actions/runs/14177411880/job/39715533887)
  * [unit_tests](https://github.com/cockroachdb/cockroach/actions/runs/14177411880/job/39715535683)
  * [linux_amd64_build](https://github.com/cockroachdb/cockroach/actions/runs/14177411880/job/39715533458)
  * [acceptance](https://github.com/cockroachdb/cockroach/actions/runs/14177411880/job/39715531791)
  * [docker_image_amd64](https://github.com/cockroachdb/cockroach/actions/runs/14177411880/job/39715533092)
  * [local_roachtest_fips](https://github.com/cockroachdb/cockroach/actions/runs/14177411880/job/39715534702)
author_association: CONTRIBUTOR