Repository: fsouza/fake-gcs-server
Pull request #307 (2020-08-12 22:04:14)

add support for MD5 checksum of object content when using fs
Copy pasted md5 and crc hash functions from `upload.go`.

I compared the results against actually uploading to GCS (and using `gsutil hash -m`) and they differ for some reason. Whilst this is somewhat annoying it isn't a problem for me as all I need is something consistent.


Patch:
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
new file mode 100644
index 0000000..32c1acf
--- /dev/null
+++ b/.github/workflows/codeql-analysis.yml
@@ -0,0 +1,54 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [main, ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [main]
+  schedule:
+    - cron: '0 23 * * 3'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2.3.2
+      with:
+        # We must fetch at least the immediate parents so that if this is
+        # a pull request then we can checkout the head.
+        fetch-depth: 2
+
+    # If this run was triggered by a pull request event, then checkout
+    # the head of the pull request instead of the merge commit.
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      # Override language selection by uncommenting this and choosing your languages
+      # with:
+      #   languages: go, javascript, csharp, python, cpp, java
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1
diff --git a/.github/workflows/goreleaser.yml b/.github/workflows/goreleaser.yml
index 879fb68..6d06b88 100644
--- a/.github/workflows/goreleaser.yml
+++ b/.github/workflows/goreleaser.yml
@@ -10,7 +10,7 @@ jobs:
 
       - name: fetch tags
         if: github.event.ref_type == 'tag'
-        run: git fetch --tags --prune --prune-tags
+        run: git fetch --tags --prune --prune-tags --force
 
       - uses: docker://goreleaser/goreleaser
         if: github.event.ref_type == 'tag'
diff --git a/README.md b/README.md
index c3bae23..ffcfd3d 100644
--- a/README.md
+++ b/README.md
@@ -16,8 +16,8 @@ fsouza/fake-gcs-server``](https://hub.docker.com/r/fsouza/fake-gcs-server)).
 
 ## Using the emulator in Docker
 
-If you want to run a standalone server (like the datastore/pubsub emulators)
-for integration tests and/or tests in other languages you may want to run the
+You can stub/mock Google Cloud Storage as a standalone server (like the datastore/pubsub emulators)
+which is ideal for integration tests and/or tests in other languages you may want to run the
 ``fake-gcs-server`` inside a Docker container:
 
 ```shell
diff --git a/examples/node/package-lock.json b/examples/node/package-lock.json
index e628a92..9de8df2 100644
--- a/examples/node/package-lock.json
+++ b/examples/node/package-lock.json
@@ -53,9 +53,9 @@
       "integrity": "sha512-EvuabjzzZ9E2+OaYf+7P9OAiiwbTxKYL0oGLnREQd+Su2NTQBpomkdlkBowFvyWsaV0d1sSGxrKpSNcrhPqbxg=="
     },
     "@google-cloud/storage": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-5.2.0.tgz",
-      "integrity": "sha512-zxHXZajtVA0Qx9IOnDUDb76mtKn5M20LKV/phmnVos7foozG9YZ6yYod90pRC/GgP3eOgxNYdt6KQcapssPsFw==",
+      "version": "5.3.0",
+      "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-5.3.0.tgz",
+      "integrity": "sha512-3t5UF3SZ14Bw2kcBHubCai6EIugU2GnQOstYWVSFuoO8IJ94RAaIOPq/dtexvQbUTpBTAGpd5smVR9WPL1mJVw==",
       "requires": {
         "@google-cloud/common": "^3.3.0",
         "@google-cloud/paginator": "^3.0.0",
@@ -168,9 +168,9 @@
       "integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA=="
     },
     "date-and-time": {
-      "version": "0.14.0",
-      "resolved": "https://registry.npmjs.org/date-and-time/-/date-and-time-0.14.0.tgz",
-      "integrity": "sha512-0wY8b90XjQkRxv3XGT8k1ffyDQOf4+T+2hiWp7rwYgoEn8OyYDsHZdnVrPlzxbwjLUY66mVBXr59eKOwpSV7lw=="
+      "version": "0.14.1",
+      "resolved": "https://registry.npmjs.org/date-and-time/-/date-and-time-0.14.1.tgz",
+      "integrity": "sha512-M4RggEH5OF2ZuCOxgOU67R6Z9ohjKbxGvAQz48vj53wLmL0bAgumkBvycR32f30pK+Og9pIR+RFDyChbaE4oLA=="
     },
     "debug": {
       "version": "4.1.1",
diff --git a/fakestorage/upload.go b/fakestorage/upload.go
index b9e95d7..0a41d34 100644
--- a/fakestorage/upload.go
+++ b/fakestorage/upload.go
@@ -72,6 +72,26 @@ func (s *Server) insertObject(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+func (s *Server) checkUploadPreconditions(w http.ResponseWriter, r *http.Request, bucketName string, objectName string) bool {
+	ifGenerationMatch := r.URL.Query().Get("ifGenerationMatch")
+
+	if ifGenerationMatch == "0" {
+		if _, err := s.backend.GetObject(bucketName, objectName); err == nil {
+			w.WriteHeader(http.StatusPreconditionFailed)
+			err := newErrorResponse(http.StatusPreconditionFailed, "Precondition failed", nil)
+			json.NewEncoder(w).Encode(err)
+			return false
+		}
+	} else if ifGenerationMatch != "" || r.URL.Query().Get("ifGenerationNotMatch") != "" {
+		w.WriteHeader(http.StatusNotImplemented)
+		err := newErrorResponse(http.StatusNotImplemented, "Precondition support not implemented", nil)
+		json.NewEncoder(w).Encode(err)
+		return false
+	}
+
+	return true
+}
+
 func (s *Server) simpleUpload(bucketName string, w http.ResponseWriter, r *http.Request) {
 	defer r.Body.Close()
 	name := r.URL.Query().Get("name")
@@ -235,6 +255,10 @@ func (s *Server) multipartUpload(bucketName string, w http.ResponseWriter, r *ht
 		objName = metadata.Name
 	}
 
+	if !s.checkUploadPreconditions(w, r, bucketName, objName) {
+		return
+	}
+
 	obj := Object{
 		BucketName:      bucketName,
 		Name:            objName,
diff --git a/go.mod b/go.mod
index ae570f1..f28ca9f 100644
--- a/go.mod
+++ b/go.mod
@@ -2,9 +2,9 @@ module github.com/fsouza/fake-gcs-server
 
 require (
 	cloud.google.com/go/storage v1.10.0
-	github.com/google/go-cmp v0.5.1
-	github.com/gorilla/handlers v1.4.2
-	github.com/gorilla/mux v1.7.4
+	github.com/google/go-cmp v0.5.2
+	github.com/gorilla/handlers v1.5.0
+	github.com/gorilla/mux v1.8.0
 	github.com/sirupsen/logrus v1.6.0
 	google.golang.org/api v0.30.0
 )
diff --git a/go.sum b/go.sum
index 69dfe52..6c1cfa5 100644
--- a/go.sum
+++ b/go.sum
@@ -68,6 +68,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
+github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -120,6 +122,8 @@ github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0 h1:pMen7vLs8nvgEYhywH3KDWJIJTeEr2ULsVWHWYHQyBs=
@@ -138,8 +142,12 @@ github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg=
 github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
+github.com/gorilla/handlers v1.5.0 h1:4wjo3sf9azi99c8hTmyaxp9y5S+pFszsy3pP0rAw/lw=
+github.com/gorilla/handlers v1.5.0/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=
 github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
 github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
diff --git a/internal/backend/fs.go b/internal/backend/fs.go
index a78b16f..1ea0d55 100644
--- a/internal/backend/fs.go
+++ b/internal/backend/fs.go
@@ -5,9 +5,12 @@
 package backend
 
 import (
+	"crypto/md5" // #nosec G501
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"hash/crc32"
 	"io/ioutil"
 	"net/url"
 	"os"
@@ -163,6 +166,37 @@ func (s *storageFS) GetObjectWithGeneration(bucketName, objectName string, gener
 	return Object{}, errors.New("not implemented: fs storage type does not support versioning yet")
 }
 
+var crc32cTable = crc32.MakeTable(crc32.Castagnoli)
+
+func crc32cChecksum(content []byte) []byte {
+	checksummer := crc32.New(crc32cTable)
+	checksummer.Write(content)
+	return checksummer.Sum(make([]byte, 0, 4))
+}
+
+func encodedChecksum(checksum []byte) string {
+	return base64.StdEncoding.EncodeToString(checksum)
+}
+
+func encodedCrc32cChecksum(content []byte) string {
+	return encodedChecksum(crc32cChecksum(content))
+}
+
+func md5Hash(b []byte) []byte {
+	/* #nosec G401 */
+	h := md5.New()
+	h.Write(b)
+	return h.Sum(nil)
+}
+
+func encodedHash(hash []byte) string {
+	return base64.StdEncoding.EncodeToString(hash)
+}
+
+func encodedMd5Hash(content []byte) string {
+	return encodedHash(md5Hash(content))
+}
+
 func (s *storageFS) getObject(bucketName, objectName string) (Object, error) {
 	encoded, err := ioutil.ReadFile(filepath.Join(s.rootDir, url.PathEscape(bucketName), url.PathEscape(objectName)))
 	if err != nil {
@@ -175,6 +209,8 @@ func (s *storageFS) getObject(bucketName, objectName string) (Object, error) {
 	}
 	obj.Name = filepath.ToSlash(objectName)
 	obj.BucketName = bucketName
+	obj.Crc32c = encodedCrc32cChecksum(obj.Content)
+	obj.Md5Hash = encodedMd5Hash(obj.Content)
 	return obj, nil
 }
 


Test patch:
diff --git a/fakestorage/upload_test.go b/fakestorage/upload_test.go
index b9913b7..30b8455 100644
--- a/fakestorage/upload_test.go
+++ b/fakestorage/upload_test.go
@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/tls"
+	"io/ioutil"
 	"net/http"
 	"reflect"
 	"strings"
@@ -137,6 +138,64 @@ func TestServerClientObjectWriterOverwrite(t *testing.T) {
 	})
 }
 
+func TestServerClientObjectWriterWithDoesNotExistPrecondition(t *testing.T) {
+	runServersTest(t, nil, func(t *testing.T, server *Server) {
+		const originalContent = "original content"
+		const originalContentType = "text/plain"
+		const bucketName = "some-bucket"
+		const objectName = "some-object-2.txt"
+
+		bucket := server.Client().Bucket(bucketName)
+		if err := bucket.Create(context.Background(), "my-project", nil); err != nil {
+			t.Fatal(err)
+		}
+
+		objHandle := bucket.Object(objectName)
+
+		firstWriter := objHandle.If(storage.Conditions{DoesNotExist: true}).NewWriter(context.Background())
+		firstWriter.ContentType = originalContentType
+		firstWriter.Write([]byte(originalContent))
+		if err := firstWriter.Close(); err != nil {
+			t.Fatal(err)
+		}
+
+		firstReader, err := objHandle.NewReader(context.Background())
+		if err != nil {
+			t.Fatal(err)
+		}
+		objectContent, err := ioutil.ReadAll(firstReader)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if string(objectContent) != originalContent {
+			t.Errorf("wrong content in the object after initial write with precondition\nwant %q\ngot  %q", originalContent, string(objectContent))
+		}
+
+		secondWriter := objHandle.If(storage.Conditions{DoesNotExist: true}).NewWriter(context.Background())
+		secondWriter.ContentType = "application/json"
+		secondWriter.Write([]byte("new content"))
+		err = secondWriter.Close()
+		if err == nil {
+			t.Fatal("expected overwriting existing object to fail, but received no error")
+		}
+		if err.Error() != "googleapi: Error 412: Precondition failed" {
+			t.Errorf("expected HTTP 412 precondition failed error, but got %v", err)
+		}
+
+		secondReader, err := objHandle.NewReader(context.Background())
+		if err != nil {
+			t.Fatal(err)
+		}
+		objectContentAfterFailedPrecondition, err := ioutil.ReadAll(secondReader)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if string(objectContentAfterFailedPrecondition) != originalContent {
+			t.Errorf("wrong content in the object after failed precondition\nwant %q\ngot  %q", originalContent, string(objectContentAfterFailedPrecondition))
+		}
+	})
+}
+
 func TestServerClientObjectWriterBucketNotFound(t *testing.T) {
 	runServersTest(t, nil, func(t *testing.T, server *Server) {
 		client := server.Client()
diff --git a/internal/backend/backend_test.go b/internal/backend/backend_test.go
index c74dfb8..7ef52fe 100644
--- a/internal/backend/backend_test.go
+++ b/internal/backend/backend_test.go
@@ -97,9 +97,12 @@ func TestObjectCRUD(t *testing.T) {
 	const bucketName = "prod-bucket"
 	const objectName = "video/hi-res/best_video_1080p.mp4"
 	content1 := []byte("content1")
-	const crc1 = "crc1"
-	const md51 = "md51"
+	const crc1 = "syeM0w=="
+	const md51 = "flXbAB0xmpSwtxNSmnVmIw=="
 	content2 := []byte("content2")
+	const crc2 = "oHd/Jw=="
+	const md52 = "7qZw9KyUHfcaO18mjr4+rA=="
+
 	for _, versioningEnabled := range []bool{true, false} {
 		versioningEnabled := versioningEnabled
 		testForStorageBackends(t, func(t *testing.T, storage Storage) {
@@ -121,7 +124,7 @@ func TestObjectCRUD(t *testing.T) {
 			initialGeneration := uploadAndCompare(t, storage, initialObject)
 
 			t.Logf("create (update) in existent case with explicit generation and versioning %t", versioningEnabled)
-			secondVersionWithGeneration := Object{BucketName: bucketName, Name: objectName, Content: content2, Generation: 1234}
+			secondVersionWithGeneration := Object{BucketName: bucketName, Name: objectName, Content: content2, Generation: 1234, Crc32c: crc2, Md5Hash: md52}
 			uploadAndCompare(t, storage, secondVersionWithGeneration)
 
 			initialObjectFromGeneration, err := storage.GetObjectWithGeneration(initialObject.BucketName, initialObject.Name, initialGeneration)