Repository: google/go-cloud
Pull request #3238 (2023-03-06 13:57:23)

blob/s3blob: adds support for per-request s3v2 Options, implementing #3214
This PR adds support in the **blob/s3blob** driver for per-request s3 options when using the underlying v2 AWS SDK.

Specifically, adds **As** support for:

 * `ListOptions.BeforeList` (V2) for the type `*[]func(*s3v2.Options)`
 * `ReaderOptions.BeforeRead` (V2) for the type `*[]func(*s3v2.Options)`

which gives the application the ability to supply per-request options when using the V2 SDK.

There does not appear to be any utility in doing the same for `BeforeWrite` because we already expose access to the `s3v2manager.Uploader` which includes the `ClientOptions` list. 


Patch:
diff --git a/.gitattributes b/.gitattributes
index 14e13ea8..53379ace 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,4 +1,6 @@
 * text eol=lf
+*.png -text
+samples/order/testdata/* -text
 /internal/website/data/examples.json  linguist-generated=true
 **/testdata/**/*.yaml    linguist-generated=true
 **/testdata/**/*.replay  linguist-generated=true
diff --git a/blob/s3blob/s3blob.go b/blob/s3blob/s3blob.go
index b697f237..c94345ed 100644
--- a/blob/s3blob/s3blob.go
+++ b/blob/s3blob/s3blob.go
@@ -44,12 +44,12 @@
 // s3blob exposes the following types for As:
 //   - Bucket: (V1) *s3.S3; (V2) *s3v2.Client
 //   - Error: (V1) awserr.Error; (V2) any error type returned by the service, notably smithy.APIError
-//   - ListObject: (V1) s3.Object for objects, s3.CommonPrefix for "directories"; (V2) typesv2.Object for objects, typesv2.CommonPrefix for "directories
-//   - ListOptions.BeforeList: (V1) *s3.ListObjectsV2Input, or *s3.ListObjectsInput
-//     when Options.UseLegacyList == true; (V2) *s3v2.ListObjectsV2Input, or *s3v2.ListObjectsInput
+//   - ListObject: (V1) s3.Object for objects, s3.CommonPrefix for "directories"; (V2) typesv2.Object for objects, typesv2.CommonPrefix for "directories"
+//   - ListOptions.BeforeList: (V1) *s3.ListObjectsV2Input or *s3.ListObjectsInput
+//     when Options.UseLegacyList == true; (V2) *s3v2.ListObjectsV2Input or *[]func(*s3v2.Options), or *s3v2.ListObjectsInput
 //     when Options.UseLegacyList == true
 //   - Reader: (V1) s3.GetObjectOutput; (V2) s3v2.GetObjectInput
-//   - ReaderOptions.BeforeRead: (V1) *s3.GetObjectInput; (V2) *s3v2.GetObjectInput
+//   - ReaderOptions.BeforeRead: (V1) *s3.GetObjectInput; (V2) *s3v2.GetObjectInput or *[]func(*s3v2.Options)
 //   - Attributes: (V1) s3.HeadObjectOutput; (V2)s3v2.HeadObjectOutput
 //   - CopyOptions.BeforeCopy: *(V1) s3.CopyObjectInput; (V2) s3v2.CopyObjectInput
 //   - WriterOptions.BeforeWrite: (V1) *s3manager.UploadInput, *s3manager.Uploader; (V2) *s3v2.PutObjectInput, *s3v2manager.Uploader
@@ -528,20 +528,24 @@ func (b *bucket) ListPaged(ctx context.Context, opts *driver.ListOptions) (*driv
 
 func (b *bucket) listObjectsV2(ctx context.Context, in *s3v2.ListObjectsV2Input, opts *driver.ListOptions) (*s3v2.ListObjectsV2Output, error) {
 	if !b.useLegacyList {
+		var varopt []func(*s3v2.Options)
 		if opts.BeforeList != nil {
 			asFunc := func(i interface{}) bool {
-				p, ok := i.(**s3v2.ListObjectsV2Input)
-				if !ok {
-					return false
+				if p, ok := i.(**s3v2.ListObjectsV2Input); ok {
+					*p = in
+					return true
 				}
-				*p = in
-				return true
+				if p, ok := i.(**[]func(*s3v2.Options)); ok {
+					*p = &varopt
+					return true
+				}
+				return false
 			}
 			if err := opts.BeforeList(asFunc); err != nil {
 				return nil, err
 			}
 		}
-		return b.clientV2.ListObjectsV2(ctx, in)
+		return b.clientV2.ListObjectsV2(ctx, in, varopt...)
 	}
 
 	// Use the legacy ListObjects request.
@@ -589,12 +593,11 @@ func (b *bucket) listObjects(ctx context.Context, in *s3.ListObjectsV2Input, opt
 	if !b.useLegacyList {
 		if opts.BeforeList != nil {
 			asFunc := func(i interface{}) bool {
-				p, ok := i.(**s3.ListObjectsV2Input)
-				if !ok {
-					return false
+				if p, ok := i.(**s3.ListObjectsV2Input); ok {
+					*p = in
+					return true
 				}
-				*p = in
-				return true
+				return false
 			}
 			if err := opts.BeforeList(asFunc); err != nil {
 				return nil, err
@@ -776,19 +779,24 @@ func (b *bucket) NewRangeReader(ctx context.Context, key string, offset, length
 			Key:    aws.String(key),
 			Range:  byteRange,
 		}
+		var varopt []func(*s3v2.Options)
 		if opts.BeforeRead != nil {
 			asFunc := func(i interface{}) bool {
 				if p, ok := i.(**s3v2.GetObjectInput); ok {
 					*p = in
 					return true
 				}
+				if p, ok := i.(**[]func(*s3v2.Options)); ok {
+					*p = &varopt
+					return true
+				}
 				return false
 			}
 			if err := opts.BeforeRead(asFunc); err != nil {
 				return nil, err
 			}
 		}
-		resp, err := b.clientV2.GetObject(ctx, in)
+		resp, err := b.clientV2.GetObject(ctx, in, varopt...)
 		if err != nil {
 			return nil, err
 		}
@@ -960,14 +968,22 @@ func (b *bucket) NewTypedWriter(ctx context.Context, key string, contentType str
 		}
 		if opts.BeforeWrite != nil {
 			asFunc := func(i interface{}) bool {
-				pu, ok := i.(**s3managerv2.Uploader)
-				if ok {
-					*pu = uploaderV2
+				// Note that since the Go CDK Blob
+				// abstraction does not expose AWS's
+				// Uploader concept, there does not
+				// appear to be any utility in
+				// exposing the options list to the v2
+				// Uploader's Upload() method.
+				// Instead, applications can
+				// manipulate the exposed *Uploader
+				// directly, including by setting
+				// ClientOptions if needed.
+				if p, ok := i.(**s3managerv2.Uploader); ok {
+					*p = uploaderV2
 					return true
 				}
-				pui, ok := i.(**s3v2.PutObjectInput)
-				if ok {
-					*pui = reqV2
+				if p, ok := i.(**s3v2.PutObjectInput); ok {
+					*p = reqV2
 					return true
 				}
 				return false


Test patch:
diff --git a/blob/s3blob/s3blob_test.go b/blob/s3blob/s3blob_test.go
index a0686bce..6e094ab5 100644
--- a/blob/s3blob/s3blob_test.go
+++ b/blob/s3blob/s3blob_test.go
@@ -129,7 +129,7 @@ func BenchmarkS3blob(b *testing.B) {
 
 const language = "nl"
 
-// verifyContentLanguage uses As to access the underlying GCS types and
+// verifyContentLanguage uses As to access the underlying AWS types and
 // read/write the ContentLanguage field.
 type verifyContentLanguage struct {
 	useV2           bool
@@ -172,8 +172,11 @@ func (v verifyContentLanguage) ErrorCheck(b *blob.Bucket, err error) error {
 
 func (v verifyContentLanguage) BeforeRead(as func(interface{}) bool) error {
 	if v.useV2 {
-		var req *s3v2.GetObjectInput
-		if !as(&req) {
+		var (
+			req  *s3v2.GetObjectInput
+			opts *[]func(*s3v2.Options)
+		)
+		if !as(&req) || !as(&opts) {
 			return errors.New("BeforeRead As failed")
 		}
 		return nil
@@ -187,8 +190,11 @@ func (v verifyContentLanguage) BeforeRead(as func(interface{}) bool) error {
 
 func (v verifyContentLanguage) BeforeWrite(as func(interface{}) bool) error {
 	if v.useV2 {
-		var req *s3v2.PutObjectInput
-		if !as(&req) {
+		var (
+			req      *s3v2.PutObjectInput
+			uploader *s3managerv2.Uploader
+		)
+		if !as(&req) || !as(&uploader) {
 			return errors.New("Writer.As failed for PutObjectInput")
 		}
 		req.ContentLanguage = aws.String(language)
@@ -233,10 +239,14 @@ func (v verifyContentLanguage) BeforeList(as func(interface{}) bool) error {
 				return errors.New("List.As failed")
 			}
 		} else {
-			var req *s3v2.ListObjectsV2Input
-			if !as(&req) {
+			var (
+				list *s3v2.ListObjectsV2Input
+				opts *[]func(*s3v2.Options)
+			)
+			if !as(&list) || !as(&opts) {
 				return errors.New("List.As failed")
 			}
+			return nil
 		}
 		return nil
 	}