Repository: getmoto/moto

Issue:
Metadata not available when using pre-signed urls
Hi all! 👋

This is a better formulation (hopefully) than #2040. What I do:
- generate a pre-signed url adding Metadata when doing it
- upload data via PUT to that generated url
- retrieve the data just uploaded

Expected: I should be able to see the metadata.
Actual result: No metadata is present.

To reproduce, run this script:

```python
# https://github.com/spulec/moto/issues/2040
import boto3
from moto import mock_s3
import requests


@mock_s3
def test_generate_presigned_url():
    conn = boto3.resource('s3', region_name='us-east-1')
    conn.create_bucket(Bucket='mybucket')
    s3 = boto3.client('s3', region_name='us-east-1')

    # Create a pre-signed url with some metadata.
    url = s3.generate_presigned_url(ClientMethod="put_object", Params={
        'Bucket': 'mybucket', 'Key': 'mykey', 'Metadata': {'venue': "123"}})

    # Use the url to upload data.
    requests.put(url, data={'Bucket': 'mybucket',
                            'Key': 'mykey', 'Body': 'myvalue'})

    # Retrieve metadata
    return s3.head_object(Bucket='mybucket', Key='mykey').get('Metadata')


# To show the bug, use test_my_model_head_object() to fill metadata. For the working example, use test_my_model_put_object().
if __name__ == '__main__':
    metadata = test_generate_presigned_url()
    assert 'venue' in metadata
    assert metadata['venue'] == '123'
```

- moto version: 1.3.7

I'll be back with a patch if time permits.


Discussion:
Possibly causing: https://github.com/localstack/localstack/issues/544
Thanks for opening. I don't believe we've added the code so that this will work so I'm going to set this as a feature request.
I've got the same issue as @sturmer, when using presigned url to upload file.  

Here is example presigned url:  
```
http://localstack:4572/trustportal-assets-local/key=d290f1ee-6c54-4b01-90e6-d701748f0852/f47c3706-52c9-4595-afb5-820b25931411/certifficate5.pdf&AWSAccessKeyId=foo&policy=eyJleHBpcmF0aW9uIjogIjIwMTktMTAtMjhUMTY6MDc6MzNaIiwgImNvbmRpdGlvbnMiOiBbeyJidWNrZXQiOiAidHJ1c3Rwb3J0YWwtYXNzZXRzLWxvY2FsIn0sIHsia2V5IjogImQyOTBmMWVlLTZjNTQtNGIwMS05MGU2LWQ3MDE3NDhmMDg1Mi9mNDdjMzcwNi01MmM5LTQ1OTUtYWZiNS04MjBiMjU5MzE0MTEvY2VydGlmZmljYXRlNS5wZGYifV19&signature=hoKKHWSsufHOL36b1u8liEO1Ex8%3D
```

After PUT file using presigned url i receive sqs message, and what i see, is that **localstack is using whole presigned url with params as key** for the file instead of parsing parameters to get data (here is excerpt from sqs message - for event fired by s3 after uploading file using above presigned url):

```
'object': {  
'key': 'key=d290f1ee-6c54-4b01-90e6-d701748f0852/f47c3706-52c9-4595-afb5-820b25931411/certifficate5.pdf&AWSAccessKeyId=foo&policy=eyJleHBpcmF0aW9uIjogIjIwMTktMTAtMjhUMTY6MDc6MzNaIiwgImNvbmRpdGlvbnMiOiBbeyJidWNrZXQiOiAidHJ1c3Rwb3J0YWwtYXNzZXRzLWxvY2FsIn0sIHsia2V5IjogImQyOTBmMWVlLTZjNTQtNGIwMS05MGU2LWQ3MDE3NDhmMDg1Mi9mNDdjMzcwNi01MmM5LTQ1OTUtYWZiNS04MjBiMjU5MzE0MTEvY2VydGlmZmljYXRlNS5wZGYifV19&signature=hoKKHWSsufHOL36b1u8liEO1Ex8%3D',  
'size': 1024,  
'eTag': 'd41d8cd98f00b204e9800998ecf8427e',
```
Other bugs with processing of that presigned url:
- size is wrong
- ETag value is wrong - looks like returned value after put is always the same: `d41d8cd98f00b204e9800998ecf8427e`
- returned status code is wrong - `204` should be returned like in aws but `200` is returned after PUT

@sturmer so if you try to get/head object with key you get error, becouse localstack has different key for your object.

Resolving patch:
diff --git a/moto/s3/responses.py b/moto/s3/responses.py
--- a/moto/s3/responses.py
+++ b/moto/s3/responses.py
@@ -1079,6 +1079,10 @@ def _key_response(self, request, full_url, headers):
             if key:
                 if not key.acl.public_read and not signed_url:
                     return 403, {}, ""
+            elif signed_url:
+                # coming in from requests.get(s3.generate_presigned_url())
+                if self._invalid_headers(request.url, dict(request.headers)):
+                    return 403, {}, S3_INVALID_PRESIGNED_PARAMETERS
 
         if hasattr(request, "body"):
             # Boto
@@ -1287,6 +1291,7 @@ def _key_response_put(self, request, body, bucket_name, query, key_name, headers
             )
             request.streaming = True
             metadata = metadata_from_headers(request.headers)
+            metadata.update(metadata_from_headers(query))
             new_key.set_metadata(metadata)
             new_key.set_acl(acl)
             new_key.website_redirect_location = request.headers.get(
@@ -1672,6 +1677,29 @@ def _key_response_post(self, request, body, bucket_name, query, key_name):
                 "Method POST had only been implemented for multipart uploads and restore operations, so far"
             )
 
+    def _invalid_headers(self, url, headers):
+        """
+        Verify whether the provided metadata in the URL is also present in the headers
+        :param url: .../file.txt&content-type=app%2Fjson&Signature=..
+        :param headers: Content-Type=app/json
+        :return: True or False
+        """
+        metadata_to_check = {
+            "content-disposition": "Content-Disposition",
+            "content-encoding": "Content-Encoding",
+            "content-language": "Content-Language",
+            "content-length": "Content-Length",
+            "content-md5": "Content-MD5",
+            "content-type": "Content-Type",
+        }
+        for url_key, header_key in metadata_to_check.items():
+            metadata_in_url = re.search(url_key + "=(.+?)(&.+$|$)", url)
+            if metadata_in_url:
+                url_value = unquote(metadata_in_url.group(1))
+                if header_key not in headers or (url_value != headers[header_key]):
+                    return True
+        return False
+
 
 S3ResponseInstance = ResponseObject(s3_backend)
 
@@ -2214,6 +2242,15 @@ def _key_response_post(self, request, body, bucket_name, query, key_name):
 </BucketEncryptionStatus>
 """
 
+S3_INVALID_PRESIGNED_PARAMETERS = """<?xml version="1.0" encoding="UTF-8"?>
+<Error>
+  <Code>SignatureDoesNotMatch</Code>
+  <Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message>
+  <RequestId>0D68A23BB2E2215B</RequestId>
+  <HostId>9Gjjt1m+cjU4OPvX9O9/8RuvnG41MRb/18Oux2o5H5MY7ISNTlXN+Dz9IG62/ILVxhAGI0qyPfg=</HostId>
+</Error>
+"""
+
 S3_NO_ENCRYPTION = """<?xml version="1.0" encoding="UTF-8"?>
 <Error>
   <Code>ServerSideEncryptionConfigurationNotFoundError</Code>
diff --git a/moto/s3/utils.py b/moto/s3/utils.py
--- a/moto/s3/utils.py
+++ b/moto/s3/utils.py
@@ -6,6 +6,7 @@
 import re
 import six
 from six.moves.urllib.parse import urlparse, unquote, quote
+from requests.structures import CaseInsensitiveDict
 import sys
 
 
@@ -62,7 +63,7 @@ def parse_region_from_url(url):
 
 
 def metadata_from_headers(headers):
-    metadata = {}
+    metadata = CaseInsensitiveDict()
     meta_regex = re.compile(r"^x-amz-meta-([a-zA-Z0-9\-_]+)$", flags=re.IGNORECASE)
     for header, value in headers.items():
         if isinstance(header, six.string_types):
@@ -75,7 +76,11 @@ def metadata_from_headers(headers):
                 # Check for special metadata that doesn't start with x-amz-meta
                 meta_key = header
             if meta_key:
-                metadata[meta_key] = headers[header]
+                metadata[meta_key] = (
+                    headers[header][0]
+                    if type(headers[header]) == list
+                    else headers[header]
+                )
     return metadata
 
 


Test patch:
diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py
--- a/tests/test_s3/test_s3.py
+++ b/tests/test_s3/test_s3.py
@@ -4583,3 +4583,106 @@ def test_encryption():
     conn.delete_bucket_encryption(Bucket="mybucket")
     with assert_raises(ClientError) as exc:
         conn.get_bucket_encryption(Bucket="mybucket")
+
+
+@mock_s3
+def test_presigned_url_restrict_parameters():
+    # Only specific params can be set
+    # Ensure error is thrown when adding custom metadata this way
+    bucket = str(uuid.uuid4())
+    key = "file.txt"
+    conn = boto3.resource("s3", region_name="us-east-1")
+    conn.create_bucket(Bucket=bucket)
+    s3 = boto3.client("s3", region_name="us-east-1")
+
+    # Create a pre-signed url with some metadata.
+    with assert_raises(botocore.exceptions.ParamValidationError) as err:
+        s3.generate_presigned_url(
+            ClientMethod="put_object",
+            Params={"Bucket": bucket, "Key": key, "Unknown": "metadata"},
+        )
+    assert str(err.exception).should.equal(
+        'Parameter validation failed:\nUnknown parameter in input: "Unknown", must be one of: ACL, Body, Bucket, CacheControl, ContentDisposition, ContentEncoding, ContentLanguage, ContentLength, ContentMD5, ContentType, Expires, GrantFullControl, GrantRead, GrantReadACP, GrantWriteACP, Key, Metadata, ServerSideEncryption, StorageClass, WebsiteRedirectLocation, SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId, SSEKMSEncryptionContext, RequestPayer, Tagging, ObjectLockMode, ObjectLockRetainUntilDate, ObjectLockLegalHoldStatus'
+    )
+
+    s3.delete_bucket(Bucket=bucket)
+
+
+@mock_s3
+def test_presigned_put_url_with_approved_headers():
+    bucket = str(uuid.uuid4())
+    key = "file.txt"
+    content = b"filecontent"
+    expected_contenttype = "app/sth"
+    conn = boto3.resource("s3", region_name="us-east-1")
+    conn.create_bucket(Bucket=bucket)
+    s3 = boto3.client("s3", region_name="us-east-1")
+
+    # Create a pre-signed url with some metadata.
+    url = s3.generate_presigned_url(
+        ClientMethod="put_object",
+        Params={"Bucket": bucket, "Key": key, "ContentType": expected_contenttype},
+    )
+
+    # Verify S3 throws an error when the header is not provided
+    response = requests.put(url, data=content)
+    response.status_code.should.equal(403)
+    str(response.content).should.contain("<Code>SignatureDoesNotMatch</Code>")
+    str(response.content).should.contain(
+        "<Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message>"
+    )
+
+    # Verify S3 throws an error when the header has the wrong value
+    response = requests.put(
+        url, data=content, headers={"Content-Type": "application/unknown"}
+    )
+    response.status_code.should.equal(403)
+    str(response.content).should.contain("<Code>SignatureDoesNotMatch</Code>")
+    str(response.content).should.contain(
+        "<Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message>"
+    )
+
+    # Verify S3 uploads correctly when providing the meta data
+    response = requests.put(
+        url, data=content, headers={"Content-Type": expected_contenttype}
+    )
+    response.status_code.should.equal(200)
+
+    # Assert the object exists
+    obj = s3.get_object(Bucket=bucket, Key=key)
+    obj["ContentType"].should.equal(expected_contenttype)
+    obj["ContentLength"].should.equal(11)
+    obj["Body"].read().should.equal(content)
+    obj["Metadata"].should.equal({})
+
+    s3.delete_object(Bucket=bucket, Key=key)
+    s3.delete_bucket(Bucket=bucket)
+
+
+@mock_s3
+def test_presigned_put_url_with_custom_headers():
+    bucket = str(uuid.uuid4())
+    key = "file.txt"
+    content = b"filecontent"
+    conn = boto3.resource("s3", region_name="us-east-1")
+    conn.create_bucket(Bucket=bucket)
+    s3 = boto3.client("s3", region_name="us-east-1")
+
+    # Create a pre-signed url with some metadata.
+    url = s3.generate_presigned_url(
+        ClientMethod="put_object",
+        Params={"Bucket": bucket, "Key": key, "Metadata": {"venue": "123"}},
+    )
+
+    # Verify S3 uploads correctly when providing the meta data
+    response = requests.put(url, data=content)
+    response.status_code.should.equal(200)
+
+    # Assert the object exists
+    obj = s3.get_object(Bucket=bucket, Key=key)
+    obj["ContentLength"].should.equal(11)
+    obj["Body"].read().should.equal(content)
+    obj["Metadata"].should.equal({"venue": "123"})
+
+    s3.delete_object(Bucket=bucket, Key=key)
+    s3.delete_bucket(Bucket=bucket)