Repository: fsouza/fake-gcs-server
Pull request #1036 (2023-01-09 17:21:04)

Fix custom-time error
Fixes https://github.com/fsouza/fake-gcs-server/issues/410.

If I now run the following python script:

```
self.client = storage.Client()
self.bucket = self.client.get_bucket(bucket_name)

blob = self.bucket.blob(blob_name)
blob.upload_from_string(value, content_type=content_type)
blob.reload()
blob.custom_time = datetime.datetime.now(datetime.timezone.utc)
blob.patch()
blob = self.bucket.get_blob(blob_name)
print(blob.custom_time)
```

It now does not print `None` but rather prints the correct time.

Let me know if there is anything I should change or add. One thing I was unsure of is that in `fakestorage/object.go`, I did not make any changes to `UpdateObject`, so it still only updates metadata rather than any other attributes like `CustomTime` or `Acl` (unlike the `PatchObject` method). Let me know if `UpdateObject` should be similarly changed.


Patch:
diff --git a/fakestorage/object.go b/fakestorage/object.go
index 4b704ec..156e5ea 100644
--- a/fakestorage/object.go
+++ b/fakestorage/object.go
@@ -42,6 +42,7 @@ type ObjectAttrs struct {
 	Created    time.Time
 	Updated    time.Time
 	Deleted    time.Time
+	CustomTime time.Time
 	Generation int64
 	Metadata   map[string]string
 }
@@ -50,24 +51,27 @@ func (o *ObjectAttrs) id() string {
 	return o.BucketName + "/" + o.Name
 }
 
+type jsonObject struct {
+	BucketName      string            `json:"bucket"`
+	Name            string            `json:"name"`
+	Size            int64             `json:"size,string"`
+	ContentType     string            `json:"contentType"`
+	ContentEncoding string            `json:"contentEncoding"`
+	Crc32c          string            `json:"crc32c,omitempty"`
+	Md5Hash         string            `json:"md5Hash,omitempty"`
+	Etag            string            `json:"etag,omitempty"`
+	ACL             []aclRule         `json:"acl,omitempty"`
+	Created         time.Time         `json:"created,omitempty"`
+	Updated         time.Time         `json:"updated,omitempty"`
+	Deleted         time.Time         `json:"deleted,omitempty"`
+	CustomTime      time.Time         `json:"customTime,omitempty"`
+	Generation      int64             `json:"generation,omitempty,string"`
+	Metadata        map[string]string `json:"metadata,omitempty"`
+}
+
 // MarshalJSON for ObjectAttrs to use ACLRule instead of storage.ACLRule
 func (o ObjectAttrs) MarshalJSON() ([]byte, error) {
-	temp := struct {
-		BucketName      string            `json:"bucket"`
-		Name            string            `json:"name"`
-		Size            int64             `json:"size,string"`
-		ContentType     string            `json:"contentType"`
-		ContentEncoding string            `json:"contentEncoding"`
-		Crc32c          string            `json:"crc32c,omitempty"`
-		Md5Hash         string            `json:"md5Hash,omitempty"`
-		Etag            string            `json:"etag,omitempty"`
-		ACL             []aclRule         `json:"acl,omitempty"`
-		Created         time.Time         `json:"created,omitempty"`
-		Updated         time.Time         `json:"updated,omitempty"`
-		Deleted         time.Time         `json:"deleted,omitempty"`
-		Generation      int64             `json:"generation,omitempty,string"`
-		Metadata        map[string]string `json:"metadata,omitempty"`
-	}{
+	temp := jsonObject{
 		BucketName:      o.BucketName,
 		Name:            o.Name,
 		ContentType:     o.ContentType,
@@ -79,6 +83,7 @@ func (o ObjectAttrs) MarshalJSON() ([]byte, error) {
 		Created:         o.Created,
 		Updated:         o.Updated,
 		Deleted:         o.Deleted,
+		CustomTime:      o.CustomTime,
 		Generation:      o.Generation,
 		Metadata:        o.Metadata,
 	}
@@ -91,22 +96,7 @@ func (o ObjectAttrs) MarshalJSON() ([]byte, error) {
 
 // UnmarshalJSON for ObjectAttrs to use ACLRule instead of storage.ACLRule
 func (o *ObjectAttrs) UnmarshalJSON(data []byte) error {
-	temp := struct {
-		BucketName      string            `json:"bucket"`
-		Name            string            `json:"name"`
-		Size            int64             `json:"size,string"`
-		ContentType     string            `json:"contentType"`
-		ContentEncoding string            `json:"contentEncoding"`
-		Crc32c          string            `json:"crc32c,omitempty"`
-		Md5Hash         string            `json:"md5Hash,omitempty"`
-		Etag            string            `json:"etag,omitempty"`
-		ACL             []aclRule         `json:"acl,omitempty"`
-		Created         time.Time         `json:"created,omitempty"`
-		Updated         time.Time         `json:"updated,omitempty"`
-		Deleted         time.Time         `json:"deleted,omitempty"`
-		Generation      int64             `json:"generation,omitempty,string"`
-		Metadata        map[string]string `json:"metadata,omitempty"`
-	}{}
+	var temp jsonObject
 	if err := json.Unmarshal(data, &temp); err != nil {
 		return err
 	}
@@ -123,6 +113,7 @@ func (o *ObjectAttrs) UnmarshalJSON(data []byte) error {
 	o.Deleted = temp.Deleted
 	o.Generation = temp.Generation
 	o.Metadata = temp.Metadata
+	o.CustomTime = temp.CustomTime
 	o.ACL = make([]storage.ACLRule, len(temp.ACL))
 	for i, ACL := range temp.ACL {
 		o.ACL[i] = storage.ACLRule(ACL)
@@ -416,6 +407,7 @@ func toBackendObjects(objects []StreamingObject) []backend.StreamingObject {
 				Created:         getCurrentIfZero(o.Created).Format(timestampFormat),
 				Deleted:         o.Deleted.Format(timestampFormat),
 				Updated:         getCurrentIfZero(o.Updated).Format(timestampFormat),
+				CustomTime:      o.CustomTime.Format(timestampFormat),
 				Generation:      o.Generation,
 				Metadata:        o.Metadata,
 			},
@@ -439,6 +431,7 @@ func bufferedObjectsToBackendObjects(objects []Object) []backend.StreamingObject
 				Created:         getCurrentIfZero(o.Created).Format(timestampFormat),
 				Deleted:         o.Deleted.Format(timestampFormat),
 				Updated:         getCurrentIfZero(o.Updated).Format(timestampFormat),
+				CustomTime:      o.CustomTime.Format(timestampFormat),
 				Generation:      o.Generation,
 				Metadata:        o.Metadata,
 			},
@@ -465,6 +458,7 @@ func fromBackendObjects(objects []backend.StreamingObject) []StreamingObject {
 				Created:         convertTimeWithoutError(o.Created),
 				Deleted:         convertTimeWithoutError(o.Deleted),
 				Updated:         convertTimeWithoutError(o.Updated),
+				CustomTime:      convertTimeWithoutError(o.CustomTime),
 				Generation:      o.Generation,
 				Metadata:        o.Metadata,
 			},
@@ -490,6 +484,7 @@ func fromBackendObjectsAttrs(objectAttrs []backend.ObjectAttrs) []ObjectAttrs {
 			Created:         convertTimeWithoutError(o.Created),
 			Deleted:         convertTimeWithoutError(o.Deleted),
 			Updated:         convertTimeWithoutError(o.Updated),
+			CustomTime:      convertTimeWithoutError(o.CustomTime),
 			Generation:      o.Generation,
 			Metadata:        o.Metadata,
 		})
@@ -945,8 +940,9 @@ func (s *Server) patchObject(r *http.Request) jsonResponse {
 	}
 
 	var payload struct {
-		Metadata map[string]string `json:"metadata"`
-		Acl      []acls
+		Metadata   map[string]string `json:"metadata"`
+		CustomTime string
+		Acl        []acls
 	}
 	err := json.NewDecoder(r.Body).Decode(&payload)
 	if err != nil {
@@ -959,6 +955,7 @@ func (s *Server) patchObject(r *http.Request) jsonResponse {
 	var attrsToUpdate backend.ObjectAttrs
 
 	attrsToUpdate.Metadata = payload.Metadata
+	attrsToUpdate.CustomTime = payload.CustomTime
 
 	if len(payload.Acl) > 0 {
 		attrsToUpdate.ACL = []storage.ACLRule{}
@@ -978,7 +975,8 @@ func (s *Server) patchObject(r *http.Request) jsonResponse {
 	defer backendObj.Close()
 
 	s.eventManager.Trigger(&backendObj, notification.EventMetadata, nil)
-	return jsonResponse{data: fromBackendObjects([]backend.StreamingObject{backendObj})[0]}
+	resp := jsonResponse{data: fromBackendObjects([]backend.StreamingObject{backendObj})[0]}
+	return resp
 }
 
 func (s *Server) updateObject(r *http.Request) jsonResponse {
diff --git a/fakestorage/response.go b/fakestorage/response.go
index 779dd58..56299f8 100644
--- a/fakestorage/response.go
+++ b/fakestorage/response.go
@@ -109,6 +109,7 @@ type objectResponse struct {
 	TimeDeleted     string                 `json:"timeDeleted,omitempty"`
 	Updated         string                 `json:"updated,omitempty"`
 	Generation      int64                  `json:"generation,string"`
+	CustomTime      string                 `json:"customTime,omitempty"`
 	Metadata        map[string]string      `json:"metadata,omitempty"`
 }
 
@@ -131,6 +132,7 @@ func newObjectResponse(obj ObjectAttrs) objectResponse {
 		TimeCreated:     formatTime(obj.Created),
 		TimeDeleted:     formatTime(obj.Deleted),
 		Updated:         formatTime(obj.Updated),
+		CustomTime:      formatTime(obj.CustomTime),
 		Generation:      obj.Generation,
 	}
 }
diff --git a/fakestorage/upload.go b/fakestorage/upload.go
index 42b8c3c..0499864 100644
--- a/fakestorage/upload.go
+++ b/fakestorage/upload.go
@@ -17,6 +17,7 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
+	"time"
 
 	"cloud.google.com/go/storage"
 	"github.com/fsouza/fake-gcs-server/internal/backend"
@@ -35,6 +36,7 @@ const (
 type multipartMetadata struct {
 	ContentType     string            `json:"contentType"`
 	ContentEncoding string            `json:"contentEncoding"`
+	CustomTime      time.Time         `json:"customTime,omitempty"`
 	Name            string            `json:"name"`
 	Metadata        map[string]string `json:"metadata"`
 }
@@ -193,6 +195,7 @@ func (s *Server) simpleUpload(bucketName string, r *http.Request) jsonResponse {
 	name := r.URL.Query().Get("name")
 	predefinedACL := r.URL.Query().Get("predefinedAcl")
 	contentEncoding := r.URL.Query().Get("contentEncoding")
+	customTime := r.URL.Query().Get("customTime")
 	if name == "" {
 		return jsonResponse{
 			status:       http.StatusBadRequest,
@@ -205,6 +208,7 @@ func (s *Server) simpleUpload(bucketName string, r *http.Request) jsonResponse {
 			Name:            name,
 			ContentType:     r.Header.Get(contentTypeHeader),
 			ContentEncoding: contentEncoding,
+			CustomTime:      convertTimeWithoutError(customTime),
 			ACL:             getObjectACL(predefinedACL),
 		},
 		Content: notImplementedSeeker{r.Body},
@@ -230,6 +234,7 @@ func (s *Server) signedUpload(bucketName string, r *http.Request) jsonResponse {
 	name := unescapeMuxVars(mux.Vars(r))["objectName"]
 	predefinedACL := r.URL.Query().Get("predefinedAcl")
 	contentEncoding := r.URL.Query().Get("contentEncoding")
+	customTime := r.URL.Query().Get("customTime")
 
 	// Load data from HTTP Headers
 	if contentEncoding == "" {
@@ -250,6 +255,7 @@ func (s *Server) signedUpload(bucketName string, r *http.Request) jsonResponse {
 			Name:            name,
 			ContentType:     r.Header.Get(contentTypeHeader),
 			ContentEncoding: contentEncoding,
+			CustomTime:      convertTimeWithoutError(customTime),
 			ACL:             getObjectACL(predefinedACL),
 			Metadata:        metaData,
 		},
@@ -337,6 +343,7 @@ func (s *Server) multipartUpload(bucketName string, r *http.Request) jsonRespons
 			Name:            objName,
 			ContentType:     contentType,
 			ContentEncoding: metadata.ContentEncoding,
+			CustomTime:      metadata.CustomTime,
 			ACL:             getObjectACL(predefinedACL),
 			Metadata:        metadata.Metadata,
 		},
@@ -378,6 +385,7 @@ func (s *Server) resumableUpload(bucketName string, r *http.Request) jsonRespons
 			Name:            objName,
 			ContentType:     metadata.ContentType,
 			ContentEncoding: contentEncoding,
+			CustomTime:      metadata.CustomTime,
 			ACL:             getObjectACL(predefinedACL),
 			Metadata:        metadata.Metadata,
 		},
diff --git a/internal/backend/fs.go b/internal/backend/fs.go
index dc70cfb..d3fde1a 100644
--- a/internal/backend/fs.go
+++ b/internal/backend/fs.go
@@ -228,12 +228,13 @@ func (s *storageFS) ListObjects(bucketName string, prefix string, versions bool)
 	defer s.mtx.RUnlock()
 
 	objects := []ObjectAttrs{}
-	// TODO: WalkDir more efficient?
 	bucketPath := filepath.Join(s.rootDir, url.PathEscape(bucketName))
-	if err := filepath.Walk(bucketPath, func(path string, info fs.FileInfo, _ error) error {
-		// Rel() should never return error since path always descend from bucketPath
+	if err := filepath.Walk(bucketPath, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
 		objName, _ := filepath.Rel(bucketPath, path)
-		// TODO: should this check path?
 		if s.mh.isSpecialFile(info.Name()) {
 			return nil
 		}
diff --git a/internal/backend/object.go b/internal/backend/object.go
index e33894b..7dc742e 100644
--- a/internal/backend/object.go
+++ b/internal/backend/object.go
@@ -28,6 +28,7 @@ type ObjectAttrs struct {
 	Created         string
 	Deleted         string
 	Updated         string
+	CustomTime      string
 	Generation      int64
 }
 


Test patch:
diff --git a/fakestorage/bucket_test.go b/fakestorage/bucket_test.go
index 2146a98..ff16623 100644
--- a/fakestorage/bucket_test.go
+++ b/fakestorage/bucket_test.go
@@ -97,7 +97,7 @@ func TestServerClientBucketAttrsAfterCreateBucket(t *testing.T) {
 func TestServerClientDeleteBucket(t *testing.T) {
 	t.Run("it deletes empty buckets", func(t *testing.T) {
 		const bucketName = "bucket-to-delete"
-		runServersTest(t, runServersOptions{}, func(t *testing.T, server *Server) {
+		runServersTest(t, runServersOptions{enableFSBackend: true}, func(t *testing.T, server *Server) {
 			server.CreateBucketWithOpts(CreateBucketOpts{Name: bucketName})
 			client := server.Client()
 			err := client.Bucket(bucketName).Delete(context.Background())
@@ -117,7 +117,7 @@ func TestServerClientDeleteBucket(t *testing.T) {
 	t.Run("it returns an error for non-empty buckets", func(t *testing.T) {
 		const bucketName = "non-empty-bucket"
 		objs := []Object{{ObjectAttrs: ObjectAttrs{BucketName: bucketName, Name: "static/js/app.js"}}}
-		runServersTest(t, runServersOptions{objs: objs}, func(t *testing.T, server *Server) {
+		runServersTest(t, runServersOptions{objs: objs, enableFSBackend: true}, func(t *testing.T, server *Server) {
 			client := server.Client()
 			err := client.Bucket(bucketName).Delete(context.Background())
 			if err == nil {
@@ -128,7 +128,7 @@ func TestServerClientDeleteBucket(t *testing.T) {
 
 	t.Run("it returns an error for unknown buckets", func(t *testing.T) {
 		const bucketName = "non-existent-bucket"
-		runServersTest(t, runServersOptions{}, func(t *testing.T, server *Server) {
+		runServersTest(t, runServersOptions{enableFSBackend: true}, func(t *testing.T, server *Server) {
 			client := server.Client()
 			err := client.Bucket(bucketName).Delete(context.Background())
 			if err == nil {
diff --git a/fakestorage/object_test.go b/fakestorage/object_test.go
index 19f270d..b04a4a8 100644
--- a/fakestorage/object_test.go
+++ b/fakestorage/object_test.go
@@ -1615,6 +1615,47 @@ func checkObjectMetadata(actual, expected map[string]string, t *testing.T) {
 	}
 }
 
+func TestServerClientObjectPatchCustomTime(t *testing.T) {
+	const (
+		bucketName  = "some-bucket"
+		objectName  = "items/data.txt"
+		content     = "some nice content"
+		contentType = "text/plain; charset=utf-8"
+	)
+	startTime := time.Now().Truncate(time.Second)
+	objs := []Object{
+		{
+			ObjectAttrs: ObjectAttrs{
+				BucketName:  bucketName,
+				Name:        objectName,
+				ContentType: contentType,
+				CustomTime:  startTime.Add(-5 * time.Hour),
+			},
+			Content: []byte(content),
+		},
+	}
+
+	runServersTest(t, runServersOptions{objs: objs}, func(t *testing.T, server *Server) {
+		client := server.Client()
+		objHandle := client.Bucket(bucketName).Object(objectName)
+
+		ctx := context.TODO()
+		_, err := objHandle.Update(ctx, storage.ObjectAttrsToUpdate{CustomTime: startTime})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		attrs, err := objHandle.Attrs(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !attrs.CustomTime.Equal(startTime) {
+			t.Errorf("unexpected custom time\nwant %q\ngot  %q", startTime.String(), attrs.CustomTime.String())
+		}
+	})
+}
+
 func TestParseRangeRequest(t *testing.T) {
 	ctx := context.TODO()
 
diff --git a/fakestorage/upload_test.go b/fakestorage/upload_test.go
index 4672e81..38e0636 100644
--- a/fakestorage/upload_test.go
+++ b/fakestorage/upload_test.go
@@ -18,6 +18,7 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 
 	"cloud.google.com/go/storage"
 	"github.com/fsouza/fake-gcs-server/internal/checksum"
@@ -70,10 +71,12 @@ func TestServerClientObjectWriter(t *testing.T) {
 				server.CreateBucketWithOpts(CreateBucketOpts{Name: test.bucketName})
 				client := server.Client()
 
+				customTime := time.Now().Truncate(time.Second).Add(-time.Hour)
 				objHandle := client.Bucket(test.bucketName).Object(test.objectName)
 				w := objHandle.NewWriter(context.Background())
 				w.ChunkSize = test.chunkSize
 				w.ContentType = contentType
+				w.CustomTime = customTime
 				w.Metadata = map[string]string{
 					"foo": "bar",
 				}
@@ -115,6 +118,9 @@ func TestServerClientObjectWriter(t *testing.T) {
 				if !reflect.DeepEqual(obj.Metadata, w.Metadata) {
 					t.Errorf("wrong meta data\nwant %+v\ngot  %+v", w.Metadata, obj.Metadata)
 				}
+				if !customTime.Equal(obj.CustomTime) {
+					t.Errorf("wrong custom time\nwant %q\ngot  %q", customTime.String(), obj.CustomTime.String())
+				}
 
 				reader, err := client.Bucket(test.bucketName).Object(test.objectName).NewReader(context.Background())
 				if err != nil {