Repository: fsouza/fake-gcs-server (go)

Issue:
bug: invalid CRC computed
In previous versions of the GCS storage client (`cloud.google.com/go/storage`, up to v1.29.0), there was a parsing bug that caused the client to skip CRC validation.

However, in v1.30.0, [a fix](https://github.com/googleapis/google-cloud-go/commit/574a86c614445f8c3f5a54446820df774c31cd47) for that bug was released, and since then I've received this error while reading:

```
storage: bad CRC on read: got 1077016486, want 2266108419
```

My hunch is the CRC calculation in `fsouza/fake-gcs-server` is incorrect (compared to how GCS does it); when the feature was added (https://github.com/fsouza/fake-gcs-server/pull/307), this comment in the description stands out:

> I compared the results against actually uploading to GCS (and using `gsutil hash -m`) and they differ for some reason. Whilst this is somewhat annoying it isn't a problem for me as all I need is something consistent.

I think this wasn't a problem then, since the header wasn't used, but now that `cloud.google.com/go/storage` has fixed that bug this is now a problem!

/cc @antonydenyer @fsouza 

Pull request:
Send X-Goog-Stored-Content-Encoding header
Fixes #1226

I've also fixed some unit tests that were passing but shouldn't have been.

Patch:
diff --git a/fakestorage/object.go b/fakestorage/object.go
index ddeb299..47994f2 100644
--- a/fakestorage/object.go
+++ b/fakestorage/object.go
@@ -887,6 +887,13 @@ func (s *Server) downloadObject(w http.ResponseWriter, r *http.Request) {
 		if obj.ContentEncoding != "" && !transcoded {
 			w.Header().Set("Content-Encoding", obj.ContentEncoding)
 		}
+		// X-Goog-Stored-Content-Encoding must be set to the original encoding,
+		// defaulting to "identity" if no encoding was set.
+		storedContentEncoding := "identity"
+		if obj.ContentEncoding != "" {
+			storedContentEncoding = obj.ContentEncoding
+		}
+		w.Header().Set("X-Goog-Stored-Content-Encoding", storedContentEncoding)
 	}
 
 	w.WriteHeader(status)
diff --git a/go.mod b/go.mod
index ed80078..542a32f 100644
--- a/go.mod
+++ b/go.mod
@@ -54,7 +54,7 @@ require (
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc
 	google.golang.org/grpc v1.56.1
-	google.golang.org/protobuf v1.30.0
+	google.golang.org/protobuf v1.31.0
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
diff --git a/go.sum b/go.sum
index be4207b..50b2b63 100644
--- a/go.sum
+++ b/go.sum
@@ -241,8 +241,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=


Test patch:
diff --git a/fakestorage/object_test.go b/fakestorage/object_test.go
index 3fcd509..38c8adf 100644
--- a/fakestorage/object_test.go
+++ b/fakestorage/object_test.go
@@ -392,7 +392,9 @@ func TestServerClientObjectTranscoding(t *testing.T) {
 				Name:            objectName,
 				ContentType:     contentType,
 				ContentEncoding: contentEncoding,
-				Crc32c:          checksum.EncodedCrc32cChecksum([]byte(content)),
+				// At storage time the CRC32C must be set to the CRC32C of the
+				// compressed data.
+				Crc32c: checksum.EncodedCrc32cChecksum(b.Bytes()),
 			},
 			Content: b.Bytes(),
 		},
diff --git a/fakestorage/upload_test.go b/fakestorage/upload_test.go
index 828f4fb..fe2ccef 100644
--- a/fakestorage/upload_test.go
+++ b/fakestorage/upload_test.go
@@ -621,10 +621,19 @@ func TestServerClientUploadWithPredefinedAclPublicRead(t *testing.T) {
 	defer server.Stop()
 	server.CreateBucketWithOpts(CreateBucketOpts{Name: "other-bucket"})
 
-	const data = "some nice content"
 	const contentType = "text/plain"
 	const contentEncoding = "gzip"
-	req, err := http.NewRequest("POST", server.URL()+"/storage/v1/b/other-bucket/o?predefinedAcl=publicRead&uploadType=media&name=some/nice/object.txt&contentEncoding="+contentEncoding, strings.NewReader(data))
+
+	const data = "some nice content"
+	// store the data compressed with gzip
+	buf := &bytes.Buffer{}
+	gzw := gzip.NewWriter(buf)
+	if _, err := gzw.Write([]byte(data)); err != nil {
+		t.Fatal(err)
+	}
+	compressed := buf.Bytes()
+
+	req, err := http.NewRequest("POST", server.URL()+"/storage/v1/b/other-bucket/o?predefinedAcl=publicRead&uploadType=media&name=some/nice/object.txt&contentEncoding="+contentEncoding, bytes.NewReader(compressed))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -666,13 +675,13 @@ func TestServerClientUploadWithPredefinedAclPublicRead(t *testing.T) {
 	if !isACLPublic(acl) {
 		t.Errorf("wrong acl\ngot %+v", acl)
 	}
-	if string(obj.Content) != data {
-		t.Errorf("wrong content\nwant %q\ngot  %q", string(obj.Content), data)
+	if string(obj.Content) != string(compressed) {
+		t.Errorf("wrong content\nwant %q\ngot  %q", string(obj.Content), string(compressed))
 	}
 	if obj.ContentType != contentType {
 		t.Errorf("wrong content type\nwant %q\ngot  %q", contentType, obj.ContentType)
 	}
-	checkChecksum(t, []byte(data), obj)
+	checkChecksum(t, compressed, obj)
 }
 
 func TestServerClientSimpleUploadNoName(t *testing.T) {