Repository: localstack/localstack

Issue:
Regression since 0.10.7: Unable to unmarshall response (Data read has a different checksum than expected.)
I have just pulled the `latest` Docker image and I'm getting

```
Unable to unmarshall response (Data read has a different checksum than expected.)
```

errors when using `S3Client#putObject` from the AWS SDK. This is almost identical to #1611 from October.

Fixing to version 0.10.7 fixes the issue. Looking at the commits since this version was released (09-Jan-2020) the following PRs are the only ones that mention S3:

- https://github.com/localstack/localstack/pull/1965
- https://github.com/localstack/localstack/pull/1976


Discussion:
Thanks for reporting @stuartleylandcole . Can you please provide a small reproducible example. The previous issue in #1611 (seems to have been related to multi-part uploads) has been fixed and should be covered with this test case: https://github.com/localstack/localstack/blob/master/tests/integration/test_s3.py#L328-L355 . Please provide some more details to help reproduce this particular issue. Thanks!
Hi @whummer. I'm not sure my specific case from #1611 was because of multi-part uploads as I was storing an empty body. Nevertheless this code will replicate the issue:

```
final S3Client s3Client = S3Client.builder()
                .httpClient(UrlConnectionHttpClient.builder().build())
                .credentialsProvider(DefaultCredentialsProvider.create())
                .region(Region.EU_WEST_1)
                .endpointOverride(URI.create("http://localhost:4572"));

s3Client.putObject(PutObjectRequest.builder()
                        .bucket(fullBucketName)
                        .key(contentDetailsId)
                        .build(),
                RequestBody.empty());
```

Same behaviour on versions 2.10.56 and 2.9.7 of the AWS SDK v2 for Java.

Possibly of use is this additional logging from the latest version of the SDK:

```
software.amazon.awssdk.core.exception.SdkClientException: Data read has a different checksum than expected. Was 0xd41d8cd98f00b204e9800998ecf8427e, but expected 0xec355f955798b44c8f00022b1d09af56
```

(Previously the log just stated `Data read has a different checksum than expected`)

Hopefully this is useful, please let me know if you need anything else!

Resolving patch:
diff --git a/localstack/services/s3/s3_listener.py b/localstack/services/s3/s3_listener.py
--- a/localstack/services/s3/s3_listener.py
+++ b/localstack/services/s3/s3_listener.py
@@ -24,6 +24,9 @@
 from localstack.services.s3 import multipart_content
 from localstack.services.generic_proxy import ProxyListener
 
+CONTENT_SHA256_HEADER = 'x-amz-content-sha256'
+STREAMING_HMAC_PAYLOAD = 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
+
 # mappings for S3 bucket notifications
 S3_NOTIFICATIONS = {}
 
@@ -418,6 +421,23 @@ def fix_creation_date(method, path, response):
     response._content = re.sub(r'([0-9])</CreationDate>', r'\1Z</CreationDate>', to_str(response._content))
 
 
+def fix_etag_for_multipart(data, headers, response):
+    # Fix for https://github.com/localstack/localstack/issues/1978
+    if headers.get(CONTENT_SHA256_HEADER) == STREAMING_HMAC_PAYLOAD:
+        try:
+            if b'chunk-signature=' not in to_bytes(data):
+                return
+            correct_hash = md5(strip_chunk_signatures(data))
+            tags = r'<ETag>%s</ETag>'
+            pattern = r'(&#34;)?([^<&]+)(&#34;)?'
+            replacement = r'\1%s\3' % correct_hash
+            response._content = re.sub(tags % pattern, tags % replacement, to_str(response.content))
+            if response.headers.get('ETag'):
+                response.headers['ETag'] = re.sub(pattern, replacement, response.headers['ETag'])
+        except Exception:
+            pass
+
+
 def remove_xml_preamble(response):
     """ Removes <?xml ... ?> from a response content """
     response._content = re.sub(r'^<\?[^\?]+\?>', '', to_str(response._content))
@@ -765,7 +785,7 @@ def forward_request(self, method, path, data, headers):
         # Related isse: https://github.com/localstack/localstack/issues/98
         # TODO we should evaluate whether to replace moto s3 with scality/S3:
         # https://github.com/scality/S3/issues/237
-        if headers.get('x-amz-content-sha256') == 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD':
+        if headers.get(CONTENT_SHA256_HEADER) == STREAMING_HMAC_PAYLOAD:
             modified_data = strip_chunk_signatures(modified_data or data)
             headers['content-length'] = headers.get('x-amz-decoded-content-length')
 
@@ -969,6 +989,7 @@ def return_response(self, method, path, data, headers, response):
             fix_delete_objects_response(bucket_name, method, parsed, data, headers, response)
             fix_metadata_key_underscores(response=response)
             fix_creation_date(method, path, response=response)
+            fix_etag_for_multipart(data, headers, response)
 
             # Remove body from PUT response on presigned URL
             # https://github.com/localstack/localstack/issues/1317
diff --git a/localstack/utils/cloudformation/template_deployer.py b/localstack/utils/cloudformation/template_deployer.py
--- a/localstack/utils/cloudformation/template_deployer.py
+++ b/localstack/utils/cloudformation/template_deployer.py
@@ -966,6 +966,7 @@ def resources_to_deploy_next(resources, stack_name):
 def get_resource_dependencies(resource_id, resource, resources):
     result = {}
     dumped = json.dumps(common.json_safe(resource))
+    dependencies = resource.get('DependsOn', [])
     for other_id, other in iteritems(resources):
         if resource != other:
             # TODO: traverse dict instead of doing string search
@@ -973,6 +974,6 @@ def get_resource_dependencies(resource_id, resource, resources):
             search2 = '{"Fn::GetAtt": ["%s", ' % other_id
             if search1 in dumped or search2 in dumped:
                 result[other_id] = other
-            if other_id in resource.get('DependsOn', []):
+            if other_id in dependencies:
                 result[other_id] = other
     return result


Test patch:
diff --git a/localstack/ext/java/src/test/java/cloud/localstack/S3FeaturesTest.java b/localstack/ext/java/src/test/java/cloud/localstack/S3FeaturesTest.java
--- a/localstack/ext/java/src/test/java/cloud/localstack/S3FeaturesTest.java
+++ b/localstack/ext/java/src/test/java/cloud/localstack/S3FeaturesTest.java
@@ -148,15 +148,20 @@ public void testListNextBatchOfObjects() {
 		assertEquals(3, someObjList.size());
 	}
 
-	private List<Object> mapFilesToSomeObject(ObjectListing objectListing) {
-		return objectListing.getObjectSummaries()
-			.stream()
-			.map(S3ObjectSummary::getKey)
-			.collect(Collectors.toList());
+	@Test
+	public void testUploadEmptyBody() {
+		AmazonS3 s3client = TestUtils.getClientS3();
+		String bucketName = UUID.randomUUID().toString();
+		String keyName = "test-key-empty";
+		s3client.createBucket(bucketName);
+		InputStream stream = new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8));
+		PutObjectRequest request = new PutObjectRequest(bucketName, keyName, stream, new ObjectMetadata());
+		PutObjectResult result = s3client.putObject(request);
+		Assert.assertEquals(result.getMetadata().getETag(), "d41d8cd98f00b204e9800998ecf8427e");
 	}
 
 	@Test
-	public void test() throws Exception {
+	public void testPresignedURLUpload() throws Exception {
 		AmazonS3 s3client = TestUtils.getClientS3();
 		Date expiration = new Date(System.currentTimeMillis() + 1000*60*5);
 		String bucketName = UUID.randomUUID().toString();
@@ -188,4 +193,16 @@ public void test() throws Exception {
 		String result = IOUtils.toString(stream.getObjectContent());
 		Assert.assertEquals(result, content);
 	}
+
+	// ----------------
+	// UTILITY METHODS
+	// ----------------
+
+	private List<Object> mapFilesToSomeObject(ObjectListing objectListing) {
+		return objectListing.getObjectSummaries()
+			.stream()
+			.map(S3ObjectSummary::getKey)
+			.collect(Collectors.toList());
+	}
+
 }