Repository: fsouza/fake-gcs-server
Pull request #538 (2021-07-20 21:48:42)

Add compose function support
Adds a [compose](https://cloud.google.com/storage/docs/json_api/v1/objects/compose) endpoint to merge the content of source objects into the destination object.

Closes: #527

nmbryant: Thanks for the feedback @fsouza, should be all set now

Patch:
diff --git a/fakestorage/object.go b/fakestorage/object.go
index c7a40eb..6432142 100644
--- a/fakestorage/object.go
+++ b/fakestorage/object.go
@@ -621,3 +621,46 @@ func (s *Server) patchObject(r *http.Request) jsonResponse {
 	}
 	return jsonResponse{data: fromBackendObjects([]backend.Object{backendObj})[0]}
 }
+
+func (s *Server) composeObject(r *http.Request) jsonResponse {
+	vars := mux.Vars(r)
+	bucketName := vars["bucketName"]
+	destinationObject := vars["destinationObject"]
+
+	var composeRequest struct {
+		SourceObjects []struct {
+			Name string
+		}
+		Destination struct {
+			Bucket      string
+			ContentType string
+			Metadata    map[string]string
+		}
+	}
+
+	decoder := json.NewDecoder(r.Body)
+	err := decoder.Decode(&composeRequest)
+	if err != nil {
+		return jsonResponse{
+			status:       http.StatusBadRequest,
+			errorMessage: "Error parsing request body",
+		}
+	}
+
+	var sourceNames []string
+	for _, n := range composeRequest.SourceObjects {
+		sourceNames = append(sourceNames, n.Name)
+	}
+
+	backendObj, err := s.backend.ComposeObject(bucketName, sourceNames, destinationObject, composeRequest.Destination.Metadata, composeRequest.Destination.ContentType)
+	if err != nil {
+		return jsonResponse{
+			status:       http.StatusInternalServerError,
+			errorMessage: "Error running compose",
+		}
+	}
+
+	obj := fromBackendObjects([]backend.Object{backendObj})[0]
+
+	return jsonResponse{data: newObjectResponse(obj)}
+}
diff --git a/fakestorage/server.go b/fakestorage/server.go
index 6f8153e..8fdeafd 100644
--- a/fakestorage/server.go
+++ b/fakestorage/server.go
@@ -199,6 +199,7 @@ func (s *Server) buildMuxer() {
 		r.Path("/b/{bucketName}/o/{objectName:.+}").Methods("DELETE").HandlerFunc(jsonToHTTPHandler(s.deleteObject))
 		r.Path("/b/{sourceBucket}/o/{sourceObject:.+}/copyTo/b/{destinationBucket}/o/{destinationObject:.+}").HandlerFunc(jsonToHTTPHandler(s.rewriteObject))
 		r.Path("/b/{sourceBucket}/o/{sourceObject:.+}/rewriteTo/b/{destinationBucket}/o/{destinationObject:.+}").HandlerFunc(jsonToHTTPHandler(s.rewriteObject))
+		r.Path("/b/{bucketName}/o/{destinationObject:.+}/compose").HandlerFunc(jsonToHTTPHandler(s.composeObject))
 	}
 
 	bucketHost := fmt.Sprintf("{bucketName}.%s", s.publicHost)
diff --git a/internal/backend/fs.go b/internal/backend/fs.go
index e0cf59f..fb47e6a 100644
--- a/internal/backend/fs.go
+++ b/internal/backend/fs.go
@@ -17,6 +17,8 @@ import (
 	"sync"
 	"syscall"
 	"time"
+
+	"github.com/fsouza/fake-gcs-server/internal/checksum"
 )
 
 // storageFS is an implementation of the backend storage that stores data on disk
@@ -218,3 +220,36 @@ func (s *storageFS) PatchObject(bucketName, objectName string, metadata map[stri
 	s.CreateObject(obj) // recreate object
 	return obj, nil
 }
+
+func (s *storageFS) ComposeObject(bucketName string, objectNames []string, destinationName string, metadata map[string]string, contentType string) (Object, error) {
+	var data []byte
+	for _, n := range objectNames {
+		obj, err := s.GetObject(bucketName, n)
+		if err != nil {
+			return Object{}, err
+		}
+		data = append(data, obj.Content...)
+	}
+
+	dest, err := s.GetObject(bucketName, destinationName)
+	if err != nil {
+		dest = Object{
+			BucketName:  bucketName,
+			Name:        destinationName,
+			ContentType: contentType,
+			Created:     time.Now().String(),
+		}
+	}
+
+	dest.Content = data
+	dest.Crc32c = checksum.EncodedCrc32cChecksum(data)
+	dest.Md5Hash = checksum.EncodedMd5Hash(data)
+	dest.Metadata = metadata
+
+	result, err := s.CreateObject(dest)
+	if err != nil {
+		return result, err
+	}
+
+	return result, nil
+}
diff --git a/internal/backend/memory.go b/internal/backend/memory.go
index 4f03f88..fb5635c 100644
--- a/internal/backend/memory.go
+++ b/internal/backend/memory.go
@@ -9,6 +9,8 @@ import (
 	"fmt"
 	"sync"
 	"time"
+
+	"github.com/fsouza/fake-gcs-server/internal/checksum"
 )
 
 const timestampFormat = "2006-01-02T15:04:05.999999Z07:00"
@@ -262,3 +264,36 @@ func (s *storageMemory) PatchObject(bucketName, objectName string, metadata map[
 	s.CreateObject(obj) // recreate object
 	return obj, nil
 }
+
+func (s *storageMemory) ComposeObject(bucketName string, objectNames []string, destinationName string, metadata map[string]string, contentType string) (Object, error) {
+	var data []byte
+	for _, n := range objectNames {
+		obj, err := s.GetObject(bucketName, n)
+		if err != nil {
+			return Object{}, err
+		}
+		data = append(data, obj.Content...)
+	}
+
+	dest, err := s.GetObject(bucketName, destinationName)
+	if err != nil {
+		dest = Object{
+			BucketName:  bucketName,
+			Name:        destinationName,
+			ContentType: contentType,
+			Created:     time.Now().String(),
+		}
+	}
+
+	dest.Content = data
+	dest.Crc32c = checksum.EncodedCrc32cChecksum(data)
+	dest.Md5Hash = checksum.EncodedMd5Hash(data)
+	dest.Metadata = metadata
+
+	result, err := s.CreateObject(dest)
+	if err != nil {
+		return result, err
+	}
+
+	return result, nil
+}
diff --git a/internal/backend/storage.go b/internal/backend/storage.go
index 1c88df8..f69d82a 100644
--- a/internal/backend/storage.go
+++ b/internal/backend/storage.go
@@ -18,6 +18,7 @@ type Storage interface {
 	GetObjectWithGeneration(bucketName, objectName string, generation int64) (Object, error)
 	DeleteObject(bucketName, objectName string) error
 	PatchObject(bucketName, objectName string, metadata map[string]string) (Object, error)
+	ComposeObject(bucketName string, objectNames []string, destinationName string, metadata map[string]string, contentType string) (Object, error)
 }
 
 type Error string


Test patch:
diff --git a/fakestorage/object_test.go b/fakestorage/object_test.go
index 8300c93..3e87efe 100644
--- a/fakestorage/object_test.go
+++ b/fakestorage/object_test.go
@@ -1411,3 +1411,149 @@ func TestParseRangeRequest(t *testing.T) {
 		})
 	}
 }
+
+func TestServiceClientComposeObject(t *testing.T) {
+	const (
+		source1Content = "some content"
+		source2Content = "other content"
+		source3Content = "third test"
+		contentType    = "text/plain; charset=utf-8"
+	)
+	u32Checksum := uint32Checksum([]byte(source1Content))
+	hash := checksum.MD5Hash([]byte(source1Content))
+
+	objs := []Object{
+		{
+			BucketName:  "first-bucket",
+			Name:        "files/source1.txt",
+			Content:     []byte(source1Content),
+			ContentType: contentType,
+			Crc32c:      checksum.EncodedChecksum(uint32ToBytes(u32Checksum)),
+			Md5Hash:     checksum.EncodedHash(hash),
+			Metadata:    map[string]string{"foo": "bar"},
+		},
+		{
+			BucketName:  "first-bucket",
+			Name:        "files/source2.txt",
+			Content:     []byte(source2Content),
+			ContentType: contentType,
+			Crc32c:      checksum.EncodedChecksum(uint32ToBytes(u32Checksum)),
+			Md5Hash:     checksum.EncodedHash(hash),
+			Metadata:    map[string]string{"foo": "bar"},
+		},
+		{
+			BucketName:  "first-bucket",
+			Name:        "files/source3.txt",
+			Content:     []byte(source3Content),
+			ContentType: contentType,
+			Crc32c:      checksum.EncodedChecksum(uint32ToBytes(u32Checksum)),
+			Md5Hash:     checksum.EncodedHash(hash),
+			Metadata:    map[string]string{"foo": "bar"},
+		},
+		{
+			BucketName:  "first-bucket",
+			Name:        "files/destination.txt",
+			Content:     []byte("test"),
+			ContentType: contentType,
+			Crc32c:      checksum.EncodedChecksum(uint32ToBytes(u32Checksum)),
+			Md5Hash:     checksum.EncodedHash(hash),
+			Metadata:    map[string]string{"foo": "bar"},
+		},
+	}
+
+	runServersTest(t, objs, func(t *testing.T, server *Server) {
+		server.CreateBucketWithOpts(CreateBucketOpts{Name: "empty-bucket"})
+		tests := []struct {
+			testCase          string
+			bucketName        string
+			destObjectName    string
+			sourceObjectNames []string
+			expectedContent   string
+		}{
+			{
+				"destination file doesn't exist",
+				"first-bucket",
+				"files/some-file.txt",
+				[]string{"files/source1.txt", "files/source2.txt"},
+				source1Content + source2Content,
+			},
+			{
+				"destination file already exists",
+				"first-bucket",
+				"files/destination.txt",
+				[]string{"files/source1.txt", "files/source2.txt"},
+				source1Content + source2Content,
+			},
+			{
+				"destination is a source",
+				"first-bucket",
+				"files/source3.txt",
+				[]string{"files/source2.txt", "files/source3.txt"},
+				source2Content + source3Content,
+			},
+		}
+		for _, test := range tests {
+			test := test
+			t.Run(test.testCase, func(t *testing.T) {
+				client := server.Client()
+
+				var sourceObjects []*storage.ObjectHandle
+				for _, n := range test.sourceObjectNames {
+					obj := client.Bucket(test.bucketName).Object(n)
+					sourceObjects = append(sourceObjects, obj)
+				}
+
+				dstObject := client.Bucket(test.bucketName).Object(test.destObjectName)
+				composer := dstObject.ComposerFrom(sourceObjects...)
+
+				composer.ContentType = contentType
+				composer.Metadata = map[string]string{"baz": "qux"}
+				attrs, err := composer.Run(context.TODO())
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				expectedChecksum := uint32Checksum([]byte(test.expectedContent))
+				expectedHash := checksum.MD5Hash([]byte(test.expectedContent))
+
+				if attrs.Bucket != test.bucketName {
+					t.Errorf("wrong bucket in compose object attrs\nwant %q\ngot  %q", test.bucketName, attrs.Bucket)
+				}
+				if attrs.Name != test.destObjectName {
+					t.Errorf("wrong name in compose object attrs\nwant %q\ngot  %q", test.destObjectName, attrs.Name)
+				}
+				if attrs.Size != int64(len(test.expectedContent)) {
+					t.Errorf("wrong size in compose object attrs\nwant %d\ngot  %d", int64(len(test.expectedContent)), attrs.Size)
+				}
+				if attrs.CRC32C != expectedChecksum {
+					t.Errorf("wrong checksum in compose object attrs\nwant %d\ngot  %d", u32Checksum, attrs.CRC32C)
+				}
+				if attrs.ContentType != contentType {
+					t.Errorf("wrong content type\nwant %q\ngot  %q", contentType, attrs.ContentType)
+				}
+				if !bytes.Equal(attrs.MD5, expectedHash) {
+					t.Errorf("wrong hash returned\nwant %d\ngot   %d", hash, attrs.MD5)
+				}
+				obj, err := server.GetObject(test.bucketName, test.destObjectName)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if string(obj.Content) != test.expectedContent {
+					t.Errorf("wrong content on object\nwant %q\ngot  %q", test.expectedContent, string(obj.Content))
+				}
+				if expect := checksum.EncodedChecksum(uint32ToBytes(expectedChecksum)); expect != obj.Crc32c {
+					t.Errorf("wrong checksum on object\nwant %s\ngot  %s", expect, obj.Crc32c)
+				}
+				if expect := checksum.EncodedHash(expectedHash); expect != obj.Md5Hash {
+					t.Errorf("wrong hash on object\nwant %s\ngot  %s", expect, obj.Md5Hash)
+				}
+				if obj.ContentType != contentType {
+					t.Errorf("wrong content type\nwant %q\ngot  %q", contentType, obj.ContentType)
+				}
+				if !reflect.DeepEqual(obj.Metadata, composer.Metadata) {
+					t.Errorf("wrong meta data\nwant %+v\ngot  %+v", composer.Metadata, obj.Metadata)
+				}
+			})
+		}
+	})
+}