Repository: fsouza/fake-gcs-server
Pull request #144 (2019-11-29 15:59:51)

Support for bucket creation time
_Again, incremental PR: to be considered after: https://github.com/fsouza/fake-gcs-server/pull/143_

This time not connected to versioning: extending the bucket representation in the memory engine to store the original bucket creation date, while in FS we are using the bucket root dir ctime to fill that field (and that is wrong, but persisting bucket metadata in that engine in a magic file looks overkilling in the current state), and extending then responses.



Patch:
diff --git a/fakestorage/bucket.go b/fakestorage/bucket.go
index 36cbd26..5cbcf69 100644
--- a/fakestorage/bucket.go
+++ b/fakestorage/bucket.go
@@ -70,7 +70,12 @@ func (s *Server) createBucketByPost(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Return the created bucket:
-	resp := newBucketResponse(name, versioning)
+	bucket, err := s.backend.GetBucket(name)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	resp := newBucketResponse(bucket)
 	json.NewEncoder(w).Encode(resp)
 }
 
@@ -94,7 +99,7 @@ func (s *Server) getBucket(w http.ResponseWriter, r *http.Request) {
 		encoder.Encode(err)
 		return
 	}
-	resp := newBucketResponse(bucket.Name, bucket.VersioningEnabled)
+	resp := newBucketResponse(bucket)
 	w.WriteHeader(http.StatusOK)
 	encoder.Encode(resp)
 }
diff --git a/fakestorage/object.go b/fakestorage/object.go
index c1cb70a..251ea58 100644
--- a/fakestorage/object.go
+++ b/fakestorage/object.go
@@ -298,7 +298,22 @@ func (s *Server) setObjectACL(w http.ResponseWriter, r *http.Request) {
 
 func (s *Server) rewriteObject(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
-	obj, err := s.GetObject(vars["sourceBucket"], vars["sourceObject"])
+	generationStr := r.FormValue("sourceGeneration")
+	var (
+		obj        Object
+		err        error
+		generation int64
+	)
+	if generationStr != "" {
+		generation, err = strconv.ParseInt(generationStr, 10, 64)
+		if err != nil {
+			http.Error(w, "Wrong generation ID", http.StatusBadRequest)
+			return
+		}
+		obj, err = s.GetObjectWithGeneration(vars["sourceBucket"], vars["sourceObject"], generation)
+	} else {
+		obj, err = s.GetObject(vars["sourceBucket"], vars["sourceObject"])
+	}
 	if err != nil {
 		http.Error(w, "not found", http.StatusNotFound)
 		return
diff --git a/fakestorage/response.go b/fakestorage/response.go
index c9e990b..8faf0db 100644
--- a/fakestorage/response.go
+++ b/fakestorage/response.go
@@ -25,28 +25,30 @@ func newListBucketsResponse(buckets []backend.Bucket) listResponse {
 		Items: make([]interface{}, len(buckets)),
 	}
 	for i, bucket := range buckets {
-		resp.Items[i] = newBucketResponse(bucket.Name, bucket.VersioningEnabled)
+		resp.Items[i] = newBucketResponse(bucket)
 	}
 	return resp
 }
 
 type bucketResponse struct {
-	Kind       string            `json:"kind"`
-	ID         string            `json:"id"`
-	Name       string            `json:"name"`
-	Versioning *bucketVersioning `json:"versioning,omitempty"`
+	Kind        string            `json:"kind"`
+	ID          string            `json:"id"`
+	Name        string            `json:"name"`
+	Versioning  *bucketVersioning `json:"versioning,omitempty"`
+	TimeCreated string            `json:"timeCreated,omitempty"`
 }
 
 type bucketVersioning struct {
 	Enabled bool `json:"enabled,omitempty"`
 }
 
-func newBucketResponse(bucketName string, versioningEnabled bool) bucketResponse {
+func newBucketResponse(bucket backend.Bucket) bucketResponse {
 	return bucketResponse{
-		Kind:       "storage#bucket",
-		ID:         bucketName,
-		Name:       bucketName,
-		Versioning: &bucketVersioning{versioningEnabled},
+		Kind:        "storage#bucket",
+		ID:          bucket.Name,
+		Name:        bucket.Name,
+		Versioning:  &bucketVersioning{bucket.VersioningEnabled},
+		TimeCreated: bucket.TimeCreated.Format(time.RFC3339),
 	}
 }
 
diff --git a/internal/backend/bucket.go b/internal/backend/bucket.go
index 40b4421..c18ad9d 100644
--- a/internal/backend/bucket.go
+++ b/internal/backend/bucket.go
@@ -1,7 +1,10 @@
 package backend
 
+import "time"
+
 // Bucket represents the bucket that is stored within the fake server.
 type Bucket struct {
 	Name              string
 	VersioningEnabled bool
+	TimeCreated       time.Time
 }
diff --git a/internal/backend/fs.go b/internal/backend/fs.go
index fa914d8..0c7a008 100644
--- a/internal/backend/fs.go
+++ b/internal/backend/fs.go
@@ -15,6 +15,8 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"syscall"
+	"time"
 )
 
 // StorageFS is an implementation of the backend storage that stores data on disk
@@ -82,12 +84,21 @@ func (s *StorageFS) ListBuckets() ([]Bucket, error) {
 	return buckets, nil
 }
 
+func timespecToTime(ts syscall.Timespec) time.Time {
+	return time.Unix(ts.Sec, ts.Nsec)
+}
+
 // GetBucket checks if a bucket exists
 func (s *StorageFS) GetBucket(name string) (Bucket, error) {
 	s.mtx.RLock()
 	defer s.mtx.RUnlock()
-	_, err := os.Stat(filepath.Join(s.rootDir, url.PathEscape(name)))
-	return Bucket{Name: name}, err
+	dirInfo, err := os.Stat(filepath.Join(s.rootDir, url.PathEscape(name)))
+	if err != nil {
+		return Bucket{}, err
+	}
+	statT := dirInfo.Sys().(*syscall.Stat_t)
+	// not true: Ctime is not created time, but not creating a file to persist this metadata, yet...
+	return Bucket{Name: name, VersioningEnabled: false, TimeCreated: timespecToTime(statT.Ctim)}, err
 }
 
 // CreateObject stores an object
diff --git a/internal/backend/memory.go b/internal/backend/memory.go
index d7aeda4..609253f 100644
--- a/internal/backend/memory.go
+++ b/internal/backend/memory.go
@@ -24,7 +24,7 @@ type bucketInMemory struct {
 }
 
 func newBucketInMemory(name string, versioningEnabled bool) bucketInMemory {
-	return bucketInMemory{Bucket{name, versioningEnabled}, []Object{}, []Object{}}
+	return bucketInMemory{Bucket{name, versioningEnabled, time.Now()}, []Object{}, []Object{}}
 }
 
 func (bm *bucketInMemory) addObject(obj Object) {
@@ -132,8 +132,8 @@ func (s *StorageMemory) ListBuckets() ([]Bucket, error) {
 	s.mtx.RLock()
 	defer s.mtx.RUnlock()
 	buckets := []Bucket{}
-	for bucket := range s.buckets {
-		buckets = append(buckets, Bucket{s.buckets[bucket].Name, s.buckets[bucket].VersioningEnabled})
+	for _, bucketInMemory := range s.buckets {
+		buckets = append(buckets, Bucket{bucketInMemory.Name, bucketInMemory.VersioningEnabled, bucketInMemory.TimeCreated})
 	}
 	return buckets, nil
 }
@@ -143,7 +143,7 @@ func (s *StorageMemory) GetBucket(name string) (Bucket, error) {
 	s.mtx.RLock()
 	defer s.mtx.RUnlock()
 	bucketInMemory, err := s.getBucketInMemory(name)
-	return Bucket{bucketInMemory.Name, bucketInMemory.VersioningEnabled}, err
+	return Bucket{bucketInMemory.Name, bucketInMemory.VersioningEnabled, bucketInMemory.TimeCreated}, err
 }
 
 func (s *StorageMemory) getBucketInMemory(name string) (bucketInMemory, error) {


Test patch:
diff --git a/fakestorage/bucket_test.go b/fakestorage/bucket_test.go
index 4dbed7b..6fe11ee 100644
--- a/fakestorage/bucket_test.go
+++ b/fakestorage/bucket_test.go
@@ -9,6 +9,7 @@ import (
 	"io/ioutil"
 	"os"
 	"testing"
+	"time"
 
 	"cloud.google.com/go/storage"
 	"google.golang.org/api/iterator"
@@ -21,7 +22,7 @@ func TestServerClientBucketAttrs(t *testing.T) {
 		{BucketName: "some-bucket", Name: "img/hi-res/party-03.jpg"},
 		{BucketName: "other-bucket", Name: "static/css/website.css"},
 	}
-
+	startTime := time.Now()
 	runServersTest(t, objs, func(t *testing.T, server *Server) {
 		client := server.Client()
 		attrs, err := client.Bucket("some-bucket").Attrs(context.Background())
@@ -35,6 +36,9 @@ func TestServerClientBucketAttrs(t *testing.T) {
 		if attrs.VersioningEnabled != false {
 			t.Errorf("wrong bucket props for %q\nexpecting no versioning by default, got it enabled", expectedName)
 		}
+		if attrs.Created.Before(startTime.Truncate(time.Second)) || time.Now().Before(attrs.Created) {
+			t.Errorf("expecting bucket creation date between test start time %v and now %v, got %v", startTime, time.Now(), attrs.Created)
+		}
 	})
 }
 
diff --git a/fakestorage/object_test.go b/fakestorage/object_test.go
index 7e9f16e..e4f082a 100644
--- a/fakestorage/object_test.go
+++ b/fakestorage/object_test.go
@@ -867,6 +867,13 @@ func TestServiceClientRewriteObject(t *testing.T) {
 			crc32c     uint32
 			md5hash    string
 		}{
+			{
+				"same bucket same file",
+				"first-bucket",
+				"files/some-file.txt",
+				checksum,
+				encodedHash(hash),
+			},
 			{
 				"same bucket",
 				"first-bucket",
@@ -931,6 +938,129 @@ func TestServiceClientRewriteObject(t *testing.T) {
 	})
 }
 
+func TestServiceClientRewriteObjectWithGenerations(t *testing.T) {
+	const (
+		overwrittenContent    = "i was there"
+		overwrittenGeneration = 123
+		latestContent         = "some content"
+		contentType           = "text/plain; charset=utf-8"
+	)
+	objs := []Object{
+		{
+			BucketName:  "first-bucket",
+			Name:        "files/some-file.txt",
+			Content:     []byte(overwrittenContent),
+			ContentType: contentType,
+			Crc32c:      encodedChecksum(uint32ToBytes(uint32Checksum([]byte(overwrittenContent)))),
+			Md5Hash:     encodedHash(md5Hash([]byte(overwrittenContent))),
+			Generation:  overwrittenGeneration,
+		},
+		{
+			BucketName:  "first-bucket",
+			Name:        "files/some-file.txt",
+			Content:     []byte(latestContent),
+			ContentType: contentType,
+			Crc32c:      encodedChecksum(uint32ToBytes(uint32Checksum([]byte(latestContent)))),
+			Md5Hash:     encodedHash(md5Hash([]byte(latestContent))),
+		},
+	}
+	tests := []struct {
+		testCase   string
+		bucketName string
+		objectName string
+		versioning bool
+		expectErr  bool
+	}{
+		{
+			"same bucket from old gen",
+			"first-bucket",
+			"files/other-file.txt",
+			true,
+			false,
+		},
+		{
+			"same bucket same file from old gen",
+			"first-bucket",
+			"files/some-file.txt",
+			true,
+			false,
+		},
+		{
+			"different bucket",
+			"empty-bucket",
+			"some/interesting/file.txt",
+			true,
+			false,
+		},
+		{
+			"no versioning, no old gen",
+			"empty-bucket",
+			"some/interesting/file.txt",
+			false,
+			true,
+		},
+	}
+	for _, test := range tests {
+		test := test
+		t.Run(test.testCase, func(t *testing.T) {
+			runServersTest(t, []Object{}, func(t *testing.T, server *Server) {
+				server.CreateBucketWithOpts(CreateBucketOpts{Name: "empty-bucket", VersioningEnabled: false})
+				server.CreateBucketWithOpts(CreateBucketOpts{Name: "first-bucket", VersioningEnabled: test.versioning})
+				for _, obj := range objs {
+					server.CreateObject(obj)
+				}
+				client := server.Client()
+				sourceObject := client.Bucket("first-bucket").Object("files/some-file.txt").Generation(overwrittenGeneration)
+				expectedContent := overwrittenContent
+				expectedChecksum := uint32Checksum([]byte(overwrittenContent))
+				expectedHash := md5Hash([]byte(overwrittenContent))
+				dstObject := client.Bucket(test.bucketName).Object(test.objectName)
+				attrs, err := dstObject.CopierFrom(sourceObject).Run(context.TODO())
+				if err != nil {
+					if test.expectErr {
+						t.Skip("we were expecting an error for this test")
+					}
+					t.Fatal(err)
+				}
+				if attrs.Bucket != test.bucketName {
+					t.Errorf("wrong bucket in copied object attrs\nwant %q\ngot  %q", test.bucketName, attrs.Bucket)
+				}
+				if attrs.Name != test.objectName {
+					t.Errorf("wrong name in copied object attrs\nwant %q\ngot  %q", test.objectName, attrs.Name)
+				}
+				if attrs.Size != int64(len(expectedContent)) {
+					t.Errorf("wrong size in copied object attrs\nwant %d\ngot  %d", len(expectedContent), attrs.Size)
+				}
+				if attrs.CRC32C != expectedChecksum {
+					t.Errorf("wrong checksum in copied object attrs\nwant %d\ngot  %d", expectedChecksum, attrs.CRC32C)
+				}
+				if attrs.ContentType != contentType {
+					t.Errorf("wrong content type\nwant %q\ngot  %q", contentType, attrs.ContentType)
+				}
+				if !bytes.Equal(attrs.MD5, expectedHash) {
+					t.Errorf("wrong hash returned\nwant %d\ngot   %d", expectedHash, attrs.MD5)
+				}
+				obj, err := server.GetObject(test.bucketName, test.objectName)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if string(obj.Content) != expectedContent {
+					t.Errorf("wrong content on object\nwant %q\ngot  %q", expectedContent, string(obj.Content))
+				}
+				if expect := encodedChecksum(uint32ToBytes(expectedChecksum)); expect != obj.Crc32c {
+					t.Errorf("wrong checksum on object\nwant %s\ngot  %s", expect, obj.Crc32c)
+				}
+				if expect := encodedHash(expectedHash); expect != obj.Md5Hash {
+					t.Errorf("wrong hash on object\nwant %s\ngot  %s", expect, obj.Md5Hash)
+				}
+				if obj.ContentType != contentType {
+					t.Errorf("wrong content type\nwant %q\ngot  %q", contentType, obj.ContentType)
+				}
+			})
+		})
+	}
+}
+
 func TestServerClientObjectDelete(t *testing.T) {
 	const (
 		bucketName = "some-bucket"
diff --git a/internal/backend/backend_test.go b/internal/backend/backend_test.go
index 432cfd2..307f8b8 100644
--- a/internal/backend/backend_test.go
+++ b/internal/backend/backend_test.go
@@ -10,6 +10,7 @@ import (
 	"os"
 	"reflect"
 	"testing"
+	"time"
 )
 
 func makeStorageBackends(t *testing.T) (map[string]Storage, func()) {
@@ -200,15 +201,17 @@ func TestBucketCreateGetList(t *testing.T) {
 			t.Fatalf("more than zero buckets found: %d, and expecting zero when starting the test", len(buckets))
 		}
 		bucketsToTest := []Bucket{
-			{"prod-bucket", false},
-			{"prod-bucket-with-versioning", true},
+			{"prod-bucket", false, time.Time{}},
+			{"prod-bucket-with-versioning", true, time.Time{}},
 		}
 		for i, bucket := range bucketsToTest {
 			_, err := storage.GetBucket(bucket.Name)
 			if err == nil {
 				t.Fatalf("bucket %s, exists before being created", bucket.Name)
 			}
+			timeBeforeCreation := time.Now().Truncate(time.Second) // we may lose precission
 			err = storage.CreateBucket(bucket.Name, bucket.VersioningEnabled)
+			timeAfterCreation := time.Now()
 			if reflect.TypeOf(storage) == reflect.TypeOf(&StorageFS{}) && bucket.VersioningEnabled {
 				if err == nil {
 					t.Fatal("fs storage should not accept creating buckets with versioning, but it's not failing")
@@ -222,8 +225,9 @@ func TestBucketCreateGetList(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			if bucketFromStorage != bucket {
-				t.Errorf("bucket %v does not have the expected props after retrieving. Expected %v", bucketFromStorage, bucket)
+			if !isBucketEquivalentTo(bucketFromStorage, bucket, timeBeforeCreation, timeAfterCreation) {
+				t.Errorf("bucket %v does not have the expected props after retrieving. Expected %v and time between %v and %v",
+					bucketFromStorage, bucket, timeBeforeCreation, timeAfterCreation)
 			}
 			buckets, err = storage.ListBuckets()
 			if err != nil {
@@ -244,6 +248,13 @@ func TestBucketCreateGetList(t *testing.T) {
 		}
 	})
 }
+
+func isBucketEquivalentTo(a, b Bucket, earliest, latest time.Time) bool {
+	return a.Name == b.Name &&
+		a.VersioningEnabled == b.VersioningEnabled &&
+		a.TimeCreated.After(earliest) && a.TimeCreated.Before(latest)
+}
+
 func TestBucketDuplication(t *testing.T) {
 	const bucketName = "prod-bucket"
 	testForStorageBackends(t, func(t *testing.T, storage Storage) {