Repository: fsouza/fake-gcs-server (go)

Issue:
Expected headers are not sent for ObjectHandle.NewReader()
The [Reader](https://godoc.org/cloud.google.com/go/storage#Reader) you get from `ObjectHandle.NewReader` has an `Attrs` field of type [ReaderObjectAttrs](https://godoc.org/cloud.google.com/go/storage#ReaderObjectAttrs).

The `storage` library fills those values in based on headers sent in the response: 

https://github.com/googleapis/google-cloud-go/blob/bd00477b309def1610fbf4cb259329deac795891/storage/reader.go#L255-L268

As far as I can tell, `fakestorage` is not setting these headers, which means that you get something like the following when you try to use them:

```
reader attrs: {Size:56 StartOffset:0 ContentType: ContentEncoding: CacheControl: LastModified:0001-01-01 00:00:00 +0000 UTC Generation:-1 Metageneration:0}
```

This library has been great, but including these headers would make it so that I can safely use the `Attrs` attribute _and_ be able to test. Thanks!




Pull request:
fakestorage: misc fixes
* Add Last-Modified and X-Goog-Generation HTTP headers for GET/HEAD object requests
* Fallback to original object's metadata on rewrite requests
* Return object generation number if available after upload
* Fix Object Generation JSON encoding to align with Go library

Signed-off-by: Prateek Malhotra <someone1@gmail.com>

Fixes #239 

Patch:
diff --git a/examples/go/go.mod b/examples/go/go.mod
index c974c57..745ca2e 100644
--- a/examples/go/go.mod
+++ b/examples/go/go.mod
@@ -4,5 +4,5 @@ go 1.13
 
 require (
 	cloud.google.com/go/storage v1.7.0
-	google.golang.org/api v0.23.0
+	google.golang.org/api v0.24.0
 )
diff --git a/examples/go/go.sum b/examples/go/go.sum
index c9d6ce3..2b1d727 100644
--- a/examples/go/go.sum
+++ b/examples/go/go.sum
@@ -313,6 +313,8 @@ google.golang.org/api v0.22.0 h1:J1Pl9P2lnmYFSJvgs70DKELqHNh8CNWXPbud4njEE2s=
 google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
 google.golang.org/api v0.23.0 h1:YlvGEOq2NA2my8cZ/9V8BcEO9okD48FlJcdqN0xJL3s=
 google.golang.org/api v0.23.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.24.0 h1:cG03eaksBzhfSIk7JRGctfp3lanklcOM/mTGvow7BbQ=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
diff --git a/examples/go/main.go b/examples/go/main.go
index f36bd80..ea37e41 100644
--- a/examples/go/main.go
+++ b/examples/go/main.go
@@ -2,6 +2,11 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// This example requires the server to be running with the flag -public-host
+// defined as storage.gcs.127.0.0.1.nip.io.
+//
+// Check the file ci/run-go-example.sh for a fully functional server + client
+// script.
 package main
 
 import (
diff --git a/fakestorage/object.go b/fakestorage/object.go
index 361526c..7a3886a 100644
--- a/fakestorage/object.go
+++ b/fakestorage/object.go
@@ -38,7 +38,7 @@ type Object struct {
 	Created    time.Time         `json:"created,omitempty"`
 	Updated    time.Time         `json:"updated,omitempty"`
 	Deleted    time.Time         `json:"deleted,omitempty"`
-	Generation int64             `json:"generation,omitempty"`
+	Generation int64             `json:"generation,omitempty,string"`
 	Metadata   map[string]string `json:"metadata,omitempty"`
 }
 
@@ -66,14 +66,19 @@ func (o *objectList) Swap(i int, j int) {
 // If the bucket within the object doesn't exist, it also creates it. If the
 // object already exists, it overrides the object.
 func (s *Server) CreateObject(obj Object) {
-	err := s.createObject(obj)
+	_, err := s.createObject(obj)
 	if err != nil {
 		panic(err)
 	}
 }
 
-func (s *Server) createObject(obj Object) error {
-	return s.backend.CreateObject(toBackendObjects([]Object{obj})[0])
+func (s *Server) createObject(obj Object) (Object, error) {
+	newObj, err := s.backend.CreateObject(toBackendObjects([]Object{obj})[0])
+	if err != nil {
+		return Object{}, err
+	}
+
+	return fromBackendObjects([]backend.Object{newObj})[0], nil
 }
 
 // ListObjects returns a sorted list of objects that match the given criteria,
@@ -320,6 +325,17 @@ func (s *Server) rewriteObject(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Only supplied metadata overwrites the new object's metdata
+	if len(metadata.Metadata) == 0 {
+		metadata.Metadata = obj.Metadata
+	}
+	if metadata.ContentType == "" {
+		metadata.ContentType = obj.ContentType
+	}
+	if metadata.ContentEncoding == "" {
+		metadata.ContentEncoding = obj.ContentEncoding
+	}
+
 	dstBucket := vars["destinationBucket"]
 	newObject := Object{
 		BucketName:      dstBucket,
@@ -361,6 +377,11 @@ func (s *Server) downloadObject(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Accept-Ranges", "bytes")
 	w.Header().Set("Content-Length", strconv.Itoa(len(content)))
 	w.Header().Set(contentTypeHeader, obj.ContentType)
+	w.Header().Set("X-Goog-Generation", strconv.FormatInt(obj.Generation, 10))
+	w.Header().Set("Last-Modified", obj.Updated.Format(http.TimeFormat))
+	if obj.ContentEncoding != "" {
+		w.Header().Set("Content-Encoding", obj.ContentEncoding)
+	}
 	w.WriteHeader(status)
 	if r.Method == http.MethodGet {
 		w.Write(content)
diff --git a/fakestorage/server.go b/fakestorage/server.go
index df5971d..4e86693 100644
--- a/fakestorage/server.go
+++ b/fakestorage/server.go
@@ -190,6 +190,10 @@ func (s *Server) buildMuxer() {
 	s.mux.Path("/download/storage/v1/b/{bucketName}/o/{objectName:.+}").Methods("GET").HandlerFunc(s.downloadObject)
 	s.mux.Path("/upload/storage/v1/b/{bucketName}/o").Methods("POST").HandlerFunc(s.insertObject)
 	s.mux.Path("/upload/resumable/{uploadId}").Methods("PUT", "POST").HandlerFunc(s.uploadFileContent)
+
+	// Signed URL Uploads
+	s.mux.Host(s.publicHost).Path("/{bucketName}/{objectName:.+}").Methods("POST", "PUT").HandlerFunc(s.insertObject)
+	s.mux.Host(bucketHost).Path("/{objectName:.+}").Methods("POST", "PUT").HandlerFunc(s.insertObject)
 }
 
 // Stop stops the server, closing all connections.
diff --git a/fakestorage/upload.go b/fakestorage/upload.go
index 0212f6d..17a1444 100644
--- a/fakestorage/upload.go
+++ b/fakestorage/upload.go
@@ -57,6 +57,17 @@ func (s *Server) insertObject(w http.ResponseWriter, r *http.Request) {
 	case "resumable":
 		s.resumableUpload(bucketName, w, r)
 	default:
+		// Support Signed URL Uploads
+		if r.URL.Query().Get("X-Goog-Algorithm") != "" {
+			switch r.Method {
+			case http.MethodPost:
+				s.resumableUpload(bucketName, w, r)
+			case http.MethodPut:
+				s.signedUpload(bucketName, w, r)
+			}
+
+			return
+		}
 		http.Error(w, "invalid uploadType", http.StatusBadRequest)
 	}
 }
@@ -85,7 +96,51 @@ func (s *Server) simpleUpload(bucketName string, w http.ResponseWriter, r *http.
 		Md5Hash:         encodedMd5Hash(data),
 		ACL:             getObjectACL(predefinedACL),
 	}
-	err = s.createObject(obj)
+	obj, err = s.createObject(obj)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	w.WriteHeader(http.StatusOK)
+	json.NewEncoder(w).Encode(obj)
+}
+
+func (s *Server) signedUpload(bucketName string, w http.ResponseWriter, r *http.Request) {
+	defer r.Body.Close()
+	name := mux.Vars(r)["objectName"]
+	predefinedACL := r.URL.Query().Get("predefinedAcl")
+	contentEncoding := r.URL.Query().Get("contentEncoding")
+
+	// Load data from HTTP Headers
+	if contentEncoding == "" {
+		contentEncoding = r.Header.Get("Content-Encoding")
+	}
+
+	metaData := make(map[string]string)
+	for key := range r.Header {
+		lowerKey := strings.ToLower(key)
+		if metaDataKey := strings.TrimPrefix(lowerKey, "x-goog-meta-"); metaDataKey != lowerKey {
+			metaData[metaDataKey] = r.Header.Get(key)
+		}
+	}
+
+	data, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	obj := Object{
+		BucketName:      bucketName,
+		Name:            name,
+		Content:         data,
+		ContentType:     r.Header.Get(contentTypeHeader),
+		ContentEncoding: contentEncoding,
+		Crc32c:          encodedCrc32cChecksum(data),
+		Md5Hash:         encodedMd5Hash(data),
+		ACL:             getObjectACL(predefinedACL),
+		Metadata:        metaData,
+	}
+	obj, err = s.createObject(obj)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
@@ -192,11 +247,12 @@ func (s *Server) multipartUpload(bucketName string, w http.ResponseWriter, r *ht
 		ACL:             getObjectACL(predefinedACL),
 		Metadata:        metadata.Metadata,
 	}
-	err = s.createObject(obj)
+	obj, err = s.createObject(obj)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
+
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(obj)
 }
@@ -303,7 +359,7 @@ func (s *Server) uploadFileContent(w http.ResponseWriter, r *http.Request) {
 	}
 	if commit {
 		s.uploads.Delete(uploadID)
-		err = s.createObject(obj)
+		obj, err = s.createObject(obj)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
diff --git a/go.mod b/go.mod
index 630084c..5f6a251 100644
--- a/go.mod
+++ b/go.mod
@@ -6,7 +6,7 @@ require (
 	github.com/gorilla/handlers v1.4.2
 	github.com/gorilla/mux v1.7.4
 	github.com/sirupsen/logrus v1.6.0
-	google.golang.org/api v0.23.0
+	google.golang.org/api v0.24.0
 )
 
 go 1.13
diff --git a/go.sum b/go.sum
index d183c63..d8ffb63 100644
--- a/go.sum
+++ b/go.sum
@@ -357,6 +357,8 @@ google.golang.org/api v0.21.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/
 google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
 google.golang.org/api v0.23.0 h1:YlvGEOq2NA2my8cZ/9V8BcEO9okD48FlJcdqN0xJL3s=
 google.golang.org/api v0.23.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.24.0 h1:cG03eaksBzhfSIk7JRGctfp3lanklcOM/mTGvow7BbQ=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
diff --git a/internal/backend/fs.go b/internal/backend/fs.go
index 3683e6a..171e537 100644
--- a/internal/backend/fs.go
+++ b/internal/backend/fs.go
@@ -46,7 +46,7 @@ func NewStorageFS(objects []Object, rootDir string) (Storage, error) {
 	}
 	s := &storageFS{rootDir: rootDir}
 	for _, o := range objects {
-		err := s.CreateObject(o)
+		_, err := s.CreateObject(o)
 		if err != nil {
 			return nil, err
 		}
@@ -108,21 +108,21 @@ func (s *storageFS) GetBucket(name string) (Bucket, error) {
 }
 
 // CreateObject stores an object as a regular file in the disk.
-func (s *storageFS) CreateObject(obj Object) error {
+func (s *storageFS) CreateObject(obj Object) (Object, error) {
 	if obj.Generation > 0 {
-		return errors.New("not implemented: fs storage type does not support objects generation yet")
+		return Object{}, errors.New("not implemented: fs storage type does not support objects generation yet")
 	}
 	s.mtx.Lock()
 	defer s.mtx.Unlock()
 	err := s.createBucket(obj.BucketName)
 	if err != nil {
-		return err
+		return Object{}, err
 	}
 	encoded, err := json.Marshal(obj)
 	if err != nil {
-		return err
+		return Object{}, err
 	}
-	return ioutil.WriteFile(filepath.Join(s.rootDir, url.PathEscape(obj.BucketName), url.PathEscape(obj.Name)), encoded, 0600)
+	return obj, ioutil.WriteFile(filepath.Join(s.rootDir, url.PathEscape(obj.BucketName), url.PathEscape(obj.Name)), encoded, 0600)
 }
 
 // ListObjects lists the objects in a given bucket with a given prefix and
diff --git a/internal/backend/memory.go b/internal/backend/memory.go
index 101fbf6..933ed5c 100644
--- a/internal/backend/memory.go
+++ b/internal/backend/memory.go
@@ -28,7 +28,7 @@ func newBucketInMemory(name string, versioningEnabled bool) bucketInMemory {
 	return bucketInMemory{Bucket{name, versioningEnabled, time.Now()}, []Object{}, []Object{}}
 }
 
-func (bm *bucketInMemory) addObject(obj Object) {
+func (bm *bucketInMemory) addObject(obj Object) Object {
 	obj.Generation = getNewGenerationIfZero(obj.Generation)
 	index := findObject(obj, bm.activeObjects, false)
 	if index >= 0 {
@@ -40,6 +40,8 @@ func (bm *bucketInMemory) addObject(obj Object) {
 	} else {
 		bm.activeObjects = append(bm.activeObjects, obj)
 	}
+
+	return obj
 }
 
 func getNewGenerationIfZero(generation int64) int64 {
@@ -155,16 +157,16 @@ func (s *storageMemory) getBucketInMemory(name string) (bucketInMemory, error) {
 }
 
 // CreateObject stores an object in the backend.
-func (s *storageMemory) CreateObject(obj Object) error {
+func (s *storageMemory) CreateObject(obj Object) (Object, error) {
 	s.mtx.Lock()
 	defer s.mtx.Unlock()
 	bucketInMemory, err := s.getBucketInMemory(obj.BucketName)
 	if err != nil {
 		bucketInMemory = newBucketInMemory(obj.BucketName, false)
 	}
-	bucketInMemory.addObject(obj)
+	newObj := bucketInMemory.addObject(obj)
 	s.buckets[obj.BucketName] = bucketInMemory
-	return nil
+	return newObj, nil
 }
 
 // ListObjects lists the objects in a given bucket with a given prefix and
diff --git a/internal/backend/storage.go b/internal/backend/storage.go
index 5b6cf51..9e0e219 100644
--- a/internal/backend/storage.go
+++ b/internal/backend/storage.go
@@ -11,7 +11,7 @@ type Storage interface {
 	CreateBucket(name string, versioningEnabled bool) error
 	ListBuckets() ([]Bucket, error)
 	GetBucket(name string) (Bucket, error)
-	CreateObject(obj Object) error
+	CreateObject(obj Object) (Object, error)
 	ListObjects(bucketName string, versions bool) ([]Object, error)
 	GetObject(bucketName, objectName string) (Object, error)
 	GetObjectWithGeneration(bucketName, objectName string, generation int64) (Object, error)


Test patch:
diff --git a/fakestorage/upload_test.go b/fakestorage/upload_test.go
index 07e0d48..b9913b7 100644
--- a/fakestorage/upload_test.go
+++ b/fakestorage/upload_test.go
@@ -191,6 +191,54 @@ func TestServerClientSimpleUpload(t *testing.T) {
 	checkChecksum(t, []byte(data), obj)
 }
 
+func TestServerClientSignedUpload(t *testing.T) {
+	server, err := NewServerWithOptions(Options{PublicHost: "127.0.0.1"})
+	if err != nil {
+		t.Fatalf("could not start server: %v", err)
+	}
+	defer server.Stop()
+	server.CreateBucketWithOpts(CreateBucketOpts{Name: "other-bucket"})
+	const data = "some nice content"
+	const contentType = "text/plain"
+	req, err := http.NewRequest("PUT", server.URL()+"/other-bucket/some/nice/object.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256", strings.NewReader(data))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	req.Header.Set("Content-Type", contentType)
+	req.Header.Set("X-Goog-Meta-Key", "Value")
+	client := http.Client{
+		Transport: &http.Transport{
+			// #nosec
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	expectedStatus := http.StatusOK
+	if resp.StatusCode != expectedStatus {
+		t.Errorf("wrong status code\nwant %d\ngot  %d", expectedStatus, resp.StatusCode)
+	}
+
+	obj, err := server.GetObject("other-bucket", "some/nice/object.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(obj.Content) != data {
+		t.Errorf("wrong content\nwant %q\ngot  %q", string(obj.Content), data)
+	}
+	if obj.ContentType != contentType {
+		t.Errorf("wrong content type\nwant %q\ngot  %q", contentType, obj.ContentType)
+	}
+	if want := map[string]string{"key": "Value"}; !reflect.DeepEqual(obj.Metadata, want) {
+		t.Errorf("wrong metadata\nwant %q\ngot  %q", want, obj.Metadata)
+	}
+	checkChecksum(t, []byte(data), obj)
+}
+
 func TestServerClientUploadWithPredefinedAclPublicRead(t *testing.T) {
 	server := NewServer(nil)
 	defer server.Stop()
diff --git a/internal/backend/backend_test.go b/internal/backend/backend_test.go
index a8a8a2e..b706f07 100644
--- a/internal/backend/backend_test.go
+++ b/internal/backend/backend_test.go
@@ -61,12 +61,12 @@ func shouldError(t *testing.T, err error) {
 
 func uploadAndCompare(t *testing.T, storage Storage, obj Object) int64 {
 	isFSStorage := reflect.TypeOf(storage) == reflect.TypeOf(&storageFS{})
-	err := storage.CreateObject(obj)
+	_, err := storage.CreateObject(obj)
 	if isFSStorage && obj.Generation != 0 {
 		t.Log("FS should not support objects generation")
 		shouldError(t, err)
 		obj.Generation = 0
-		err = storage.CreateObject(obj)
+		_, err = storage.CreateObject(obj)
 	}
 	noError(t, err)
 	activeObj, err := storage.GetObject(obj.BucketName, obj.Name)
@@ -185,7 +185,7 @@ func TestObjectQueryErrors(t *testing.T) {
 				return
 			}
 			validObject := Object{BucketName: bucketName, Name: "random-object", Content: []byte("random-content")}
-			err = storage.CreateObject(validObject)
+			_, err = storage.CreateObject(validObject)
 			noError(t, err)
 			_, err = storage.GetObjectWithGeneration(validObject.BucketName, validObject.Name, 33333)
 			shouldError(t, err)