Repository: google/go-cloud
Pull request #3486 (2024-09-11 00:23:34)

s3blob/blob: support additional endpoint query parameters
This commit adds the following query parameters for AWS:

1. `dualstack`
2. `fips`
3. `accelerate` (S3-only)

This avoids the need for users to specify `endpoint`. For example, if my AWS S3 bucket is `my-bucket` in `us-east-1`, and you want to enable transfer acceleration, dual-stack support, and/or FIPS, you would need to configure `endpoint` with one of the following:

1. `my-bucket.s3-accelerate.amazonaws.com`
2. `my-bucket.s3-accelerate.dualstack.amazonaws.com`
3. `my-bucket.s3-fips.us-gov-east-1.amazonaws.com`
4. `my-bucket.s3-fips.dualstack.us-east-1.amazonaws.com`

For example, for the last option, users can use `s3://my-bucket?fips=true&dualstack=true`.

Closes https://github.com/google/go-cloud/issues/3484


codecov[bot]: ## [Codecov](https://app.codecov.io/gh/google/go-cloud/pull/3486?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=google) Report
Attention: Patch coverage is `75.00000%` with `10 lines` in your changes missing coverage. Please review.
> Project coverage is 73.38%. Comparing base [(`49a4d98`)](https://app.codecov.io/gh/google/go-cloud/commit/49a4d98c75ad4a4efafda913d91bbda7ff829931?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=google) to head [(`f71c3c4`)](https://app.codecov.io/gh/google/go-cloud/commit/f71c3c4b9e8899345c6dc9e720b9ab99d6e064db?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=google).
> Report is 2 commits behind head on master.

| [Files with missing lines](https://app.codecov.io/gh/google/go-cloud/pull/3486?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=google) | Patch % | Lines |
|---|---|---|
| [aws/aws.go](https://app.codecov.io/gh/google/go-cloud/pull/3486?src=pr&el=tree&filepath=aws%2Faws.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=google#diff-YXdzL2F3cy5nbw==) | 72.41% | [4 Missing and 4 partials :warning: ](https://app.codecov.io/gh/google/go-cloud/pull/3486?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=google) |
| [blob/s3blob/s3blob.go](https://app.codecov.io/gh/google/go-cloud/pull/3486?src=pr&el=tree&filepath=blob%2Fs3blob%2Fs3blob.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=google#diff-YmxvYi9zM2Jsb2IvczNibG9iLmdv) | 81.81% | [1 Missing and 1 partial :warning: ](https://app.codecov.io/gh/google/go-cloud/pull/3486?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=google) |

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master    #3486      +/-   ##
==========================================
+ Coverage   73.14%   73.38%   +0.23%     
==========================================
  Files         113      113              
  Lines       14983    15161     +178     
==========================================
+ Hits        10960    11126     +166     
- Misses       3260     3264       +4     
- Partials      763      771       +8     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/google/go-cloud/pull/3486?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=google).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=google).


Patch:
diff --git a/aws/aws.go b/aws/aws.go
index 0f2d9c61..2e4cbafd 100644
--- a/aws/aws.go
+++ b/aws/aws.go
@@ -26,6 +26,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/google/wire"
 )
@@ -81,6 +82,8 @@ func (co ConfigOverrider) ClientConfig(serviceName string, cfgs ...*aws.Config)
 //   - endpoint: The endpoint URL (hostname only or fully qualified URI); sets aws.Config.Endpoint.
 //   - disableSSL: A value of "true" disables SSL when sending requests; sets aws.Config.DisableSSL.
 //   - s3ForcePathStyle: A value of "true" forces the request to use path-style addressing; sets aws.Config.S3ForcePathStyle.
+//   - dualstack: A value of "true" enables dual stack (IPv4 and IPv6) endpoints
+//   - fips: A value of "true" enables the use of FIPS endpoints
 func ConfigFromURLParams(q url.Values) (*aws.Config, error) {
 	var cfg aws.Config
 	for param, values := range q {
@@ -102,6 +105,20 @@ func ConfigFromURLParams(q url.Values) (*aws.Config, error) {
 				return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err)
 			}
 			cfg.S3ForcePathStyle = aws.Bool(b)
+		case "dualstack":
+			b, err := strconv.ParseBool(value)
+			if err != nil {
+				return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err)
+			}
+			cfg.UseDualStack = aws.Bool(b)
+		case "fips":
+			b, err := strconv.ParseBool(value)
+			if err != nil {
+				return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err)
+			}
+			if b {
+				cfg.UseFIPSEndpoint = endpoints.FIPSEndpointStateEnabled
+			}
 		case "awssdk":
 			// ignore, should be handled before this
 		default:
@@ -178,6 +195,8 @@ func NewDefaultV2Config(ctx context.Context) (awsv2.Config, error) {
 //   - profile: The shared config profile to use; sets SharedConfigProfile.
 //   - endpoint: The AWS service endpoint to send HTTP request.
 //   - hostname_immutable: Make the hostname immutable, only works if endpoint is also set.
+//   - dualstack: A value of "true" enables dual stack (IPv4 and IPv6) endpoints.
+//   - fips: A value of "true" enables the use of FIPS endpoints.
 func V2ConfigFromURLParams(ctx context.Context, q url.Values) (awsv2.Config, error) {
 	var endpoint string
 	var hostnameImmutable bool
@@ -197,6 +216,22 @@ func V2ConfigFromURLParams(ctx context.Context, q url.Values) (awsv2.Config, err
 			endpoint = value
 		case "profile":
 			opts = append(opts, awsv2cfg.WithSharedConfigProfile(value))
+		case "dualstack":
+			dualStack, err := strconv.ParseBool(value)
+			if err != nil {
+				return awsv2.Config{}, fmt.Errorf("invalid value for dualstack: %w", err)
+			}
+			if dualStack {
+				opts = append(opts, awsv2cfg.WithUseDualStackEndpoint(awsv2.DualStackEndpointStateEnabled))
+			}
+		case "fips":
+			fips, err := strconv.ParseBool(value)
+			if err != nil {
+				return awsv2.Config{}, fmt.Errorf("invalid value for fips: %w", err)
+			}
+			if fips {
+				opts = append(opts, awsv2cfg.WithUseFIPSEndpoint(awsv2.FIPSEndpointStateEnabled))
+			}
 		case "awssdk":
 			// ignore, should be handled before this
 		default:
@@ -215,5 +250,6 @@ func V2ConfigFromURLParams(ctx context.Context, q url.Values) (awsv2.Config, err
 			})
 		opts = append(opts, awsv2cfg.WithEndpointResolverWithOptions(customResolver))
 	}
+
 	return awsv2cfg.LoadDefaultConfig(ctx, opts...)
 }
diff --git a/blob/s3blob/s3blob.go b/blob/s3blob/s3blob.go
index e2d42daa..7ed47e44 100644
--- a/blob/s3blob/s3blob.go
+++ b/blob/s3blob/s3blob.go
@@ -130,6 +130,11 @@ const Scheme = "s3"
 // Use "awssdk=v1" to force using AWS SDK v1, "awssdk=v2" to force using AWS SDK v2,
 // or anything else to accept the default.
 //
+// The following S3-specific query options are also supported:
+//   - ssetype: The type of server side encryption used (AES256, aws:kms, aws:kms:dsse)
+//   - kmskeyid: The KMS key ID for server side encryption
+//   - accelerate: A value of "true" uses the S3 Transfer Accleration endpoints
+//
 // For V1, see gocloud.dev/aws/ConfigFromURLParams for supported query parameters
 // for overriding the aws.Session from the URL.
 // For V2, see gocloud.dev/aws/V2ConfigFromURLParams.
@@ -145,8 +150,9 @@ type URLOpener struct {
 }
 
 const (
-	sseTypeParamKey  = "ssetype"
-	kmsKeyIdParamKey = "kmskeyid"
+	sseTypeParamKey    = "ssetype"
+	kmsKeyIdParamKey   = "kmskeyid"
+	accelerateParamKey = "accelerate"
 )
 
 func toServerSideEncryptionType(value string) (typesv2.ServerSideEncryption, error) {
@@ -178,12 +184,24 @@ func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket
 		o.Options.KMSEncryptionID = kmsKeyID
 	}
 
+	accelerate := false
+	if accelerateParam := q.Get(accelerateParamKey); accelerateParam != "" {
+		q.Del(accelerateParamKey)
+		var err error
+		accelerate, err = strconv.ParseBool(accelerateParam)
+		if err != nil {
+			return nil, fmt.Errorf("invalid value for %q: %v", accelerateParamKey, err)
+		}
+	}
+
 	if o.UseV2 {
 		cfg, err := gcaws.V2ConfigFromURLParams(ctx, q)
 		if err != nil {
 			return nil, fmt.Errorf("open bucket %v: %v", u, err)
 		}
-		clientV2 := s3v2.NewFromConfig(cfg)
+		clientV2 := s3v2.NewFromConfig(cfg, func(o *s3v2.Options) {
+			o.UseAccelerate = accelerate
+		})
 
 		return OpenBucketV2(ctx, clientV2, u.Host, &o.Options)
 	}
@@ -194,6 +212,8 @@ func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket
 	if err != nil {
 		return nil, fmt.Errorf("open bucket %v: %v", u, err)
 	}
+
+	overrideCfg.S3UseAccelerate = &accelerate
 	configProvider.Configs = append(configProvider.Configs, overrideCfg)
 
 	return OpenBucket(ctx, configProvider, u.Host, &o.Options)


Test patch:
diff --git a/aws/aws_test.go b/aws/aws_test.go
index 29dbee1a..8ad2984e 100644
--- a/aws/aws_test.go
+++ b/aws/aws_test.go
@@ -184,6 +184,10 @@ func TestV2ConfigFromURLParams(t *testing.T) {
 				HostnameImmutable: true,
 			},
 		},
+		{
+			name:  "FIPS and dual stack",
+			query: url.Values{"fips": {"true"}, "dualstack": {"true"}},
+		},
 		// Can't test "profile", since AWS validates that the profile exists.
 	}
 
diff --git a/blob/s3blob/s3blob_test.go b/blob/s3blob/s3blob_test.go
index 381156d0..8d54c521 100644
--- a/blob/s3blob/s3blob_test.go
+++ b/blob/s3blob/s3blob_test.go
@@ -476,6 +476,26 @@ func TestOpenBucketFromURL(t *testing.T) {
 		{"s3://mybucket?awssdk=v2", false},
 		// OK, use KMS Server Side Encryption
 		{"s3://mybucket?ssetype=aws:kms&kmskeyid=arn:aws:us-east-1:12345:key/1-a-2-b", false},
+		// OK, use S3 Transfer acceleration and dual stack endpoints
+		{"s3://mybucket?accelerate=true&dualstack=true", false},
+		// OK, use FIPS endpoints
+		{"s3://mybucket?fips=true", false},
+		// OK, use S3 Transfer accleration and dual stack endpoints (v1)
+		{"s3://mybucket?awssdk=v1&accelerate=true&dualstack=true", false},
+		// OK, use FIPS endpoints (v1)
+		{"s3://mybucket?awssdk=v1&fips=true", false},
+		// Invalid accelerate (v1)
+		{"s3://mybucket?awssdk=v1&accelerate=bogus", true},
+		// Invalid accelerate (v2)
+		{"s3://mybucket?accelerate=bogus", true},
+		// Invalid FIPS (v1)
+		{"s3://mybucket?awssdk=v1&fips=bogus", true},
+		// Invalid FIPS (v2)
+		{"s3://mybucket?fips=bogus", true},
+		// Invalid dualstack (v1)
+		{"s3://mybucket?awssdk=v1&dualstack=bad", true},
+		// Invalid dualstack (v2)
+		{"s3://mybucket?dualstack=bad", true},
 		// Invalid ssetype
 		{"s3://mybucket?ssetype=aws:notkmsoraes&kmskeyid=arn:aws:us-east-1:12345:key/1-a-2-b", true},
 		// Invalid parameter together with a valid one.