Repository: localstack/localstack

Issue:
S3: Input is expected to be encoded in multiple of 2 bytes but found: 31
### Report/investigation

Yesterday I pulled the `latest` Localstack image from [Docker Hub](https://hub.docker.com/r/localstack/localstack/tags) and a handful of tests starting failing with the following error:

```
java.lang.IllegalArgumentException: Input is expected to be encoded in multiple of 2 bytes but found: 31

	at software.amazon.awssdk.utils.internal.Base16Codec.decode(Base16Codec.java:56)
	at software.amazon.awssdk.utils.internal.Base16Lower.decode(Base16Lower.java:65)
	at software.amazon.awssdk.services.s3.checksums.ChecksumsEnabledValidator.validatePutObjectChecksum(ChecksumsEnabledValidator.java:144)
	at software.amazon.awssdk.services.s3.internal.handlers.SyncChecksumValidationInterceptor.afterUnmarshalling(SyncChecksumValidationInterceptor.java:95)
	at software.amazon.awssdk.core.interceptor.ExecutionInterceptorChain.lambda$afterUnmarshalling$9(ExecutionInterceptorChain.java:161)
	at software.amazon.awssdk.core.interceptor.ExecutionInterceptorChain.reverseForEach(ExecutionInterceptorChain.java:219)
	at software.amazon.awssdk.core.interceptor.ExecutionInterceptorChain.afterUnmarshalling(ExecutionInterceptorChain.java:161)
	at software.amazon.awssdk.core.internal.handler.BaseClientHandler.lambda$runAfterUnmarshallingInterceptors$2(BaseClientHandler.java:146)
	at software.amazon.awssdk.core.internal.handler.BaseClientHandler.lambda$composeResponseFunctions$6(BaseClientHandler.java:273)
	at software.amazon.awssdk.core.internal.handler.BaseClientHandler.lambda$successTransformationResponseHandler$4(BaseClientHandler.java:219)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.HandleResponseStage.execute(HandleResponseStage.java:40)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.HandleResponseStage.execute(HandleResponseStage.java:30)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage.execute(ApiCallAttemptTimeoutTrackingStage.java:74)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage.execute(ApiCallAttemptTimeoutTrackingStage.java:43)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage.execute(TimeoutExceptionHandlingStage.java:78)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage.execute(TimeoutExceptionHandlingStage.java:40)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage$RetryExecutor.doExecute(RetryableStage.java:114)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage$RetryExecutor.execute(RetryableStage.java:87)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:63)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:43)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:57)
	at software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:37)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.executeWithTimer(ApiCallTimeoutTrackingStage.java:81)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:61)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:43)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:37)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:26)
	at software.amazon.awssdk.core.internal.http.AmazonSyncHttpClient$RequestExecutionBuilderImpl.execute(AmazonSyncHttpClient.java:198)
	at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.invoke(BaseSyncClientHandler.java:122)
	at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.doExecute(BaseSyncClientHandler.java:148)
	at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.execute(BaseSyncClientHandler.java:102)
	at software.amazon.awssdk.core.client.handler.SdkSyncClientHandler.execute(SdkSyncClientHandler.java:45)
	at software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler.execute(AwsSyncClientHandler.java:55)
	at software.amazon.awssdk.services.s3.DefaultS3Client.putObject(DefaultS3Client.java:7376)
	at <SNIP>
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:628)
	at org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:117)
	at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeTestMethod$7(TestMethodTestDescriptor.java:184)
	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
	at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeTestMethod(TestMethodTestDescriptor.java:180)
	at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:127)
	at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:68)
	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:135)
	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:125)
	at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:135)
	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:123)
	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
	at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:122)
	at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:80)
	at java.util.ArrayList.forEach(ArrayList.java:1257)
	at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:125)
	at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:135)
	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:123)
	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
	at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:122)
	at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:80)
	at java.util.ArrayList.forEach(ArrayList.java:1257)
	at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:125)
	at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:135)
	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:123)
	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
	at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:122)
	at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:80)
	at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
	at org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
	at org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
	at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:229)
	at org.junit.platform.launcher.core.DefaultLauncher.lambda$execute$6(DefaultLauncher.java:197)
	at org.junit.platform.launcher.core.DefaultLauncher.withInterceptedStreams(DefaultLauncher.java:211)
	at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:191)
	at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:128)
	at com.intellij.junit5.JUnit5IdeaTestRunner.startRunnerWithArgs(JUnit5IdeaTestRunner.java:69)
	at com.intellij.rt.execution.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:47)
	at com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:242)
	at com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:70)
```

This is the `validatePutObjectChecksum()` method:

```
    public static void validatePutObjectChecksum(PutObjectResponse response, ExecutionAttributes executionAttributes) {
        SdkChecksum checksum = executionAttributes.getAttribute(CHECKSUM);

        if (response.eTag() != null) {
            String contentMd5 = BinaryUtils.toBase64(checksum.getChecksumBytes());
            byte[] digest = BinaryUtils.fromBase64(contentMd5);
            byte[] ssHash = Base16Lower.decode(response.eTag().replace("\"", ""));

            if (!Arrays.equals(digest, ssHash)) {
                throw SdkClientException.create(
                    String.format("Data read has a different checksum than expected. Was 0x%s, but expected 0x%s",
                                  BinaryUtils.toHex(digest), BinaryUtils.toHex(ssHash)));
            }
        }
    }
```

It is checking the response ETag. I know there have been some changes to S3 ETags in Localstack as I reported https://github.com/localstack/localstack/issues/1978 recently. However I can't see any obvious commits since then that touch this functionality. Fixing to Localstack version 0.10.7 means all my S3 tests pass which is surprising as I would expect to hit #1978 again.

### Replication

`S3Client` configuration:

```
final S3Client s3Client = S3Client.builder()
                .httpClient(UrlConnectionHttpClient.builder().build())
                .credentialsProvider(DefaultCredentialsProvider.create())
                .region(Region.EU_WEST_1)
                .endpointOverride(URI.create("http://localhost:4572"));
```

Two different `putObject` operations causing this:

```
final byte[] bytes = "some text".getBytes(StandardCharsets.UTF_8);
s3Client.putObject(PutObjectRequest.builder()
                .bucket("my-bucket")
                .key("binary-content")
                .contentEncoding(StandardCharsets.UTF_8.toString())
                .contentType("application/octet-stream")
                .build()),
                RequestBody.fromBytes(bytes)
        );
```

```
final String text = "Very important text";
s3Client.putObject(PutObjectRequest.builder()
                .bucket("my-bucket")
                .key("text-content")
                .contentEncoding(StandardCharsets.UTF_8.toString())
                .contentType("text/plain")
                .build()),
                RequestBody.fromString(text)
        );
```

The only (obvious) differences are the content type (`text/plain` vs `application/octet-stream`) and how the `RequestBody` is built (`fromBytes()` vs `fromString()`).

### Thanks

Finally I just want to thank you for an absolutely invaluable project, my team and I could not have been achieved half of what we've done without Localstack. Just wanted to make sure you were aware that your efforts were appreciated as all I seem to do is report issues here 😃 


Discussion:
I also got this error today, for now we returned back to version 0.10.7.

Resolving patch:
diff --git a/localstack/services/generic_proxy.py b/localstack/services/generic_proxy.py
--- a/localstack/services/generic_proxy.py
+++ b/localstack/services/generic_proxy.py
@@ -103,6 +103,9 @@ def __init__(self, request, client_address, server):
             BaseHTTPRequestHandler.__init__(self, request, client_address, server)
         except SSLError as e:
             LOG.warning('SSL error when handling request: %s' % e)
+        except Exception as e:
+            if 'cannot read from timed out object' not in str(e):
+                LOG.warning('Unknown error: %s' % e)
 
     def parse_request(self):
         result = BaseHTTPRequestHandler.parse_request(self)
@@ -322,7 +325,8 @@ def is_full_url(url):
         except Exception as e:
             trace = str(traceback.format_exc())
             conn_errors = ('ConnectionRefusedError', 'NewConnectionError',
-                           'Connection aborted', 'Unexpected EOF', 'Connection reset by peer')
+                           'Connection aborted', 'Unexpected EOF', 'Connection reset by peer',
+                           'cannot read from timed out object')
             conn_error = any(e in trace for e in conn_errors)
             error_msg = 'Error forwarding request: %s %s' % (e, trace)
             if 'Broken pipe' in trace:
diff --git a/localstack/services/s3/s3_listener.py b/localstack/services/s3/s3_listener.py
--- a/localstack/services/s3/s3_listener.py
+++ b/localstack/services/s3/s3_listener.py
@@ -430,7 +430,7 @@ def fix_etag_for_multipart(data, headers, response):
             correct_hash = md5(strip_chunk_signatures(data))
             tags = r'<ETag>%s</ETag>'
             pattern = r'(&#34;)?([^<&]+)(&#34;)?'
-            replacement = r'\1%s\3' % correct_hash
+            replacement = r'\g<1>%s\g<3>' % correct_hash
             response._content = re.sub(tags % pattern, tags % replacement, to_str(response.content))
             if response.headers.get('ETag'):
                 response.headers['ETag'] = re.sub(pattern, replacement, response.headers['ETag'])
diff --git a/localstack/services/sqs/sqs_listener.py b/localstack/services/sqs/sqs_listener.py
--- a/localstack/services/sqs/sqs_listener.py
+++ b/localstack/services/sqs/sqs_listener.py
@@ -37,6 +37,7 @@
     'DelaySeconds', 'MaximumMessageSize', 'MessageRetentionPeriod', 'Policy', 'RedrivePolicy']
 
 # maps queue URLs to attributes set via the API
+# TODO: add region as first level in the map
 QUEUE_ATTRIBUTES = {}
 
 


Test patch: