Repository: fsouza/fake-gcs-server
Pull request #18 (2019-03-14 23:01:03)

fakestorage: add support for MD5 checksum of object content
We'd like to use this library to test some GCS related code that relies on the MD5 object attributes. From my testing, I believe this to be correct and I greatly cribbed off the crc32c implementation. Happy to implement any improvements suggested though.

Thanks for the library!


Patch:
diff --git a/fakestorage/object.go b/fakestorage/object.go
index 30bb7ac..bc1d472 100644
--- a/fakestorage/object.go
+++ b/fakestorage/object.go
@@ -22,7 +22,8 @@ type Object struct {
 	Name       string `json:"name"`
 	Content    []byte `json:"-"`
 	// Crc32c checksum of Content. calculated by server when it's upload methods are used.
-	Crc32c string `json:"crc32c,omitempty"`
+	Crc32c  string `json:"crc32c,omitempty"`
+	Md5Hash string `json:"md5hash,omitempty"`
 }
 
 func (o *Object) id() string {
@@ -102,6 +103,7 @@ func toBackendObjects(objects []Object) []backend.Object {
 			Name:       o.Name,
 			Content:    o.Content,
 			Crc32c:     o.Crc32c,
+			Md5Hash:    o.Md5Hash,
 		})
 	}
 	return backendObjects
@@ -115,6 +117,7 @@ func fromBackendObjects(objects []backend.Object) []Object {
 			Name:       o.Name,
 			Content:    o.Content,
 			Crc32c:     o.Crc32c,
+			Md5Hash:    o.Md5Hash,
 		})
 	}
 	return backendObjects
@@ -187,6 +190,7 @@ func (s *Server) rewriteObject(w http.ResponseWriter, r *http.Request) {
 		Name:       vars["destinationObject"],
 		Content:    append([]byte(nil), obj.Content...),
 		Crc32c:     obj.Crc32c,
+		Md5Hash:    obj.Md5Hash,
 	}
 	s.CreateObject(newObject)
 	w.Header().Set("Content-Type", "application/json")
diff --git a/fakestorage/response.go b/fakestorage/response.go
index 0572374..7c40bf1 100644
--- a/fakestorage/response.go
+++ b/fakestorage/response.go
@@ -59,17 +59,19 @@ type objectResponse struct {
 	Bucket string `json:"bucket"`
 	Size   int64  `json:"size,string"`
 	// Crc32c: CRC32c checksum, same as in google storage client code
-	Crc32c string `json:"crc32c,omitempty"`
+	Crc32c  string `json:"crc32c,omitempty"`
+	Md5Hash string `json:"md5hash,omitempty"`
 }
 
 func newObjectResponse(obj Object) objectResponse {
 	return objectResponse{
-		Kind:   "storage#object",
-		ID:     obj.id(),
-		Bucket: obj.BucketName,
-		Name:   obj.Name,
-		Size:   int64(len(obj.Content)),
-		Crc32c: obj.Crc32c,
+		Kind:    "storage#object",
+		ID:      obj.id(),
+		Bucket:  obj.BucketName,
+		Name:    obj.Name,
+		Size:    int64(len(obj.Content)),
+		Crc32c:  obj.Crc32c,
+		Md5Hash: obj.Md5Hash,
 	}
 }
 
diff --git a/fakestorage/upload.go b/fakestorage/upload.go
index b681784..8f3c85a 100644
--- a/fakestorage/upload.go
+++ b/fakestorage/upload.go
@@ -5,6 +5,7 @@
 package fakestorage
 
 import (
+	"crypto/md5" // #nosec G501
 	"crypto/rand"
 	"encoding/json"
 	"fmt"
@@ -61,7 +62,7 @@ func (s *Server) simpleUpload(bucketName string, w http.ResponseWriter, r *http.
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-	obj := Object{BucketName: bucketName, Name: name, Content: data, Crc32c: encodedCrc32cChecksum(data)}
+	obj := Object{BucketName: bucketName, Name: name, Content: data, Crc32c: encodedCrc32cChecksum(data), Md5Hash: encodedMd5Hash(data)}
 	err = s.createObject(obj)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -87,6 +88,21 @@ func encodedCrc32cChecksum(content []byte) string {
 	return encodedChecksum(crc32cChecksum(content))
 }
 
+func md5Hash(b []byte) []byte {
+	/* #nosec G401 */
+	h := md5.New()
+	h.Write(b)
+	return h.Sum(nil)
+}
+
+func encodedHash(hash []byte) string {
+	return base64.StdEncoding.EncodeToString(hash)
+}
+
+func encodedMd5Hash(content []byte) string {
+	return encodedHash(md5Hash(content))
+}
+
 func (s *Server) multipartUpload(bucketName string, w http.ResponseWriter, r *http.Request) {
 	defer r.Body.Close()
 	_, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
@@ -114,7 +130,7 @@ func (s *Server) multipartUpload(bucketName string, w http.ResponseWriter, r *ht
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-	obj := Object{BucketName: bucketName, Name: metadata.Name, Content: content, Crc32c: encodedCrc32cChecksum(content)}
+	obj := Object{BucketName: bucketName, Name: metadata.Name, Content: content, Crc32c: encodedCrc32cChecksum(content), Md5Hash: encodedMd5Hash(content)}
 	err = s.createObject(obj)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -165,6 +181,7 @@ func (s *Server) uploadFileContent(w http.ResponseWriter, r *http.Request) {
 	objLength := len(obj.Content)
 	obj.Content = append(obj.Content, content...)
 	obj.Crc32c = encodedCrc32cChecksum(obj.Content)
+	obj.Md5Hash = encodedMd5Hash(obj.Content)
 	if contentRange := r.Header.Get("Content-Range"); contentRange != "" {
 		commit, err = parseRange(contentRange, objLength, len(content), w)
 		if err != nil {
diff --git a/internal/backend/object.go b/internal/backend/object.go
index bbcf24d..e0ca2b1 100644
--- a/internal/backend/object.go
+++ b/internal/backend/object.go
@@ -10,6 +10,7 @@ type Object struct {
 	Name       string `json:"-"`
 	Content    []byte
 	Crc32c     string
+	Md5Hash    string
 }
 
 // ID is useful for comparing objects


Test patch:
diff --git a/fakestorage/object_test.go b/fakestorage/object_test.go
index c2ac38c..b6aa046 100644
--- a/fakestorage/object_test.go
+++ b/fakestorage/object_test.go
@@ -5,6 +5,7 @@
 package fakestorage
 
 import (
+	"bytes"
 	"context"
 	"encoding/binary"
 	"hash/crc32"
@@ -35,8 +36,9 @@ func TestServerClientObjectAttrs(t *testing.T) {
 		content    = "some nice content"
 	)
 	checksum := uint32Checksum([]byte(content))
+	hash := md5Hash([]byte(content))
 	objs := []Object{
-		{BucketName: bucketName, Name: objectName, Content: []byte(content), Crc32c: encodedChecksum(uint32ToBytes(checksum))},
+		{BucketName: bucketName, Name: objectName, Content: []byte(content), Crc32c: encodedChecksum(uint32ToBytes(checksum)), Md5Hash: encodedHash(hash)},
 	}
 
 	runServersTest(t, objs, func(t *testing.T, server *Server) {
@@ -58,6 +60,9 @@ func TestServerClientObjectAttrs(t *testing.T) {
 		if attrs.CRC32C != checksum {
 			t.Errorf("wrong checksum returned\nwant %d\ngot   %d", checksum, attrs.CRC32C)
 		}
+		if !bytes.Equal(attrs.MD5, hash) {
+			t.Errorf("wrong hash returned\nwant %d\ngot   %d", hash, attrs.MD5)
+		}
 	})
 }
 
@@ -410,8 +415,9 @@ func TestServiceClientListObjectsBucketNotFound(t *testing.T) {
 func TestServiceClientRewriteObject(t *testing.T) {
 	const content = "some content"
 	checksum := uint32Checksum([]byte(content))
+	hash := md5Hash([]byte(content))
 	objs := []Object{
-		{BucketName: "first-bucket", Name: "files/some-file.txt", Content: []byte(content), Crc32c: encodedChecksum(uint32ToBytes(checksum))},
+		{BucketName: "first-bucket", Name: "files/some-file.txt", Content: []byte(content), Crc32c: encodedChecksum(uint32ToBytes(checksum)), Md5Hash: encodedHash(hash)},
 	}
 
 	runServersTest(t, objs, func(t *testing.T, server *Server) {
@@ -421,18 +427,21 @@ func TestServiceClientRewriteObject(t *testing.T) {
 			bucketName string
 			objectName string
 			crc32c     uint32
+			md5hash    string
 		}{
 			{
 				"same bucket",
 				"first-bucket",
 				"files/other-file.txt",
 				checksum,
+				encodedHash(hash),
 			},
 			{
 				"different bucket",
 				"empty-bucket",
 				"some/interesting/file.txt",
 				checksum,
+				encodedHash(hash),
 			},
 		}
 		for _, test := range tests {
@@ -457,6 +466,9 @@ func TestServiceClientRewriteObject(t *testing.T) {
 				if attrs.CRC32C != checksum {
 					t.Errorf("wrong checksum in copied object attrs\nwant %d\ngot  %d", checksum, attrs.CRC32C)
 				}
+				if !bytes.Equal(attrs.MD5, hash) {
+					t.Errorf("wrong hash returned\nwant %d\ngot   %d", hash, attrs.MD5)
+				}
 				obj, err := server.GetObject(test.bucketName, test.objectName)
 				if err != nil {
 					t.Fatal(err)
@@ -467,6 +479,9 @@ func TestServiceClientRewriteObject(t *testing.T) {
 				if expect := encodedChecksum(uint32ToBytes(checksum)); expect != obj.Crc32c {
 					t.Errorf("wrong checksum on object\nwant %s\ngot  %s", expect, obj.Crc32c)
 				}
+				if expect := encodedHash(hash); expect != obj.Md5Hash {
+					t.Errorf("wrong hash on object\nwant %s\ngot  %s", expect, obj.Md5Hash)
+				}
 			})
 		}
 	})
diff --git a/fakestorage/upload_test.go b/fakestorage/upload_test.go
index 6582845..6cfca7d 100644
--- a/fakestorage/upload_test.go
+++ b/fakestorage/upload_test.go
@@ -5,6 +5,7 @@
 package fakestorage
 
 import (
+	"bytes"
 	"context"
 	"crypto/tls"
 	"net/http"
@@ -18,6 +19,7 @@ func TestServerClientObjectWriter(t *testing.T) {
 	const baseContent = "some nice content"
 	content := strings.Repeat(baseContent+"\n", googleapi.MinUploadChunkSize)
 	checksum := uint32Checksum([]byte(content))
+	hash := md5Hash([]byte(content))
 
 	runServersTest(t, nil, func(t *testing.T, server *Server) {
 		var tests = []struct {
@@ -72,6 +74,12 @@ func TestServerClientObjectWriter(t *testing.T) {
 				if base64Checksum := encodedChecksum(uint32ToBytes(checksum)); obj.Crc32c != base64Checksum {
 					t.Errorf("wrong obj.Crc32c returned\nwant %s\ngot %s", base64Checksum, obj.Crc32c)
 				}
+				if returnedHash := w.Attrs().MD5; !bytes.Equal(returnedHash, hash) {
+					t.Errorf("wrong writer.Attrs() hash returned\nwant %d\ngot  %d", hash, returnedHash)
+				}
+				if stringHash := encodedHash(hash); obj.Md5Hash != stringHash {
+					t.Errorf("wrong obj.Md5Hash returned\nwant %s\ngot %s", stringHash, obj.Md5Hash)
+				}
 			})
 		}
 	})
diff --git a/internal/backend/backend_test.go b/internal/backend/backend_test.go
index a36da82..6abc296 100644
--- a/internal/backend/backend_test.go
+++ b/internal/backend/backend_test.go
@@ -60,6 +60,7 @@ func TestObjectCRUD(t *testing.T) {
 	const objectName = "video/hi-res/best_video_1080p.mp4"
 	content1 := []byte("content1")
 	const crc1 = "crc1"
+	const md51 = "md51"
 	content2 := []byte("content2")
 	testForStorageBackends(t, func(t *testing.T, storage Storage) {
 		// Get in non-existent case
@@ -69,7 +70,7 @@ func TestObjectCRUD(t *testing.T) {
 		err = storage.DeleteObject(bucketName, objectName)
 		shouldError(t, err, "object successfully delete before being created")
 		// Create in non-existent case
-		noError(t, storage.CreateObject(Object{BucketName: bucketName, Name: objectName, Content: content1, Crc32c: crc1}))
+		noError(t, storage.CreateObject(Object{BucketName: bucketName, Name: objectName, Content: content1, Crc32c: crc1, Md5Hash: md51}))
 		// Get in existent case
 		obj, err := storage.GetObject(bucketName, objectName)
 		noError(t, err)
@@ -82,6 +83,9 @@ func TestObjectCRUD(t *testing.T) {
 		if obj.Crc32c != crc1 {
 			t.Errorf("wrong crc\n want %q\ngot  %q", crc1, obj.Crc32c)
 		}
+		if obj.Md5Hash != md51 {
+			t.Errorf("wrong md5\n want %q\ngot  %q", md51, obj.Md5Hash)
+		}
 		if !bytes.Equal(obj.Content, content1) {
 			t.Errorf("wrong object content\n want %q\ngot  %q", content1, obj.Content)
 		}