Repository: getmoto/moto

Issue:
mock_sqs throws exception when StringValue is XML
When I run the following code which uses boto3 and an AWS SQS it works as expected, I get a JSON response containing XML.

    import boto3
    import time
    
    queue_url = 'https://sqs.eu-west-2.amazonaws.com/[redacted]'
    
    sqs = boto3.client("sqs", region_name="eu-west-2")
    data = {
        'Payload':
            {
                'DataType': 'String',
                'StringValue': '<?xml version="1.0" encoding="UTF-8"?><feed xmlns="http://www.w3.org/2005/Atom"/>'
            },
        'FeedType':
            {
                'DataType': 'String', 'StringValue': 'Atom'
            }
    }
    
    response = sqs.send_message(
        QueueUrl=queue_url,
        MessageBody='NSWSS Atom Feed',
        MessageAttributes=data
    )
    print(response)
    
    time.sleep(5)
    
    fetch_response = sqs.receive_message(
        QueueUrl=queue_url,
        MessageAttributeNames=('Payload', 'FeedType'),
        MaxNumberOfMessages=1,
        VisibilityTimeout=0
    )
    
    print(fetch_response)

it prints:


    {'MD5OfMessageBody': '62f853b97bc8b350d6e868c52bf3e5e0', 'MD5OfMessageAttributes': '61ca54311f9c73c111b62ff02f407e1f', 'MessageId': 'ce092897-4a5e-4902-9070-90c907e8c19f', 'ResponseMetadata': {'RequestId': 'ee0544b6-c1e5-56a7-97fa-1d723231c767', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': 'ee0544b6-c1e5-56a7-97fa-1d723231c767', 'date': 'Wed, 22 Sep 2021 09:04:20 GMT', 'content-type': 'text/xml', 'content-length': '459'}, 'RetryAttempts': 0}}

    {'Messages': [{'MessageId': 'ce092897-4a5e-4902-9070-90c907e8c19f', 'ReceiptHandle': 'AQEBMti9hU0LykJfTkrOhJHkoa0sq5ho09frwDgixQDzrnV7F2pCWwkEx1wJSQiJOjcCdFhGvd0xATysW8onRxfgSFELF1Wh+Wew7+lwAKp3/rC9AlSVeU72e/QnPnrLm1PfbOzIzCHbYO9CkUqgDVTsEaHqBq6f+aufQjP+u3MkJF8Kd4ZrapVSWULxz0kfFh2JNfTBEgRmmiOp+MzsjgTPtOSxCmgsss+h48Njx2Z9zSk8AgIIulyiyplNZwCcIRr2cGYTmaIt8D4pyRI2U7WJdfZJ3roG8evy3wauVptjksZY1+JjGvZMzCNY+WS9I8pI1ZwQowJhuEVddWVxJ31c0YgKdXXzPQ3wuiDZn1Ki2RblxOuSWCZU2qbnw5ev6wVYYE9uXQGvKoFbbd7Uhzmn8g==', 'MD5OfBody': '62f853b97bc8b350d6e868c52bf3e5e0', 'Body': 'NSWSS Atom Feed', 'MD5OfMessageAttributes': '61ca54311f9c73c111b62ff02f407e1f', 'MessageAttributes': {'FeedType': {'StringValue': 'Atom', 'DataType': 'String'}, 'Payload': {'StringValue': '<?xml version="1.0" encoding="UTF-8"?><feed xmlns="http://www.w3.org/2005/Atom"/>', 'DataType': 'String'}}}], 'ResponseMetadata': {'RequestId': '7c35d819-e42a-5727-b441-81dd17f3f6e7', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '7c35d819-e42a-5727-b441-81dd17f3f6e7', 'date': 'Wed, 22 Sep 2021 09:04:25 GMT', 'content-type': 'text/xml', 'content-length': '1327'}, 'RetryAttempts': 0}}

However when I run similar code with moto, mock_sqs I get an exception and a XML response:

    import boto3
    import time
    
    from moto import mock_sqs
    
    with mock_sqs():
        sqs = boto3.client("sqs", region_name="eu-west-2")
        queue_url = sqs.create_queue(QueueName="test-queue")["QueueUrl"]
        data = {
            'Payload':
                {
                    'DataType': 'String',
                    'StringValue': '<?xml version="1.0" encoding="UTF-8"?><feed xmlns="http://www.w3.org/2005/Atom"/>'
                },
            'FeedType':
                {
                    'DataType': 'String', 'StringValue': 'Atom'
                }
        }
    
        response = sqs.send_message(
            QueueUrl=queue_url,
            MessageBody='NSWSS Atom Feed',
            MessageAttributes=data
        )
        print(response)

    time.sleep(5)

    fetch_response = sqs.receive_message(
        QueueUrl=queue_url,
        MessageAttributeNames=('Payload', 'FeedType'),
        MaxNumberOfMessages=1,
        VisibilityTimeout=0
    )

    print(fetch_response)

The following is printed:


    {'MD5OfMessageBody': '62f853b97bc8b350d6e868c52bf3e5e0', 'MD5OfMessageAttributes': '61ca54311f9c73c111b62ff02f407e1f', 'MessageId': '5f92f101-431a-8a92-0d1c-fce671a53204', 'ResponseMetadata': {'RequestId': 'YF7QLBEKZY0A0QUYDMMCKXUR7LM8HQMWZUICHFPUVPXQEZL2FIF1', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'amazon.com', 'x-amzn-requestid': 'YF7QLBEKZY0A0QUYDMMCKXUR7LM8HQMWZUICHFPUVPXQEZL2FIF1', 'x-amz-crc32': '2210275125'}, 'RetryAttempts': 0}}

    Traceback (most recent call last):
    File "C:\Users\dan.small\nswws-public-api-venv\lib\site-packages\botocore\parsers.py", line 479, in _parse_xml_string_to_dom
    parser.feed(xml_string)
    xml.etree.ElementTree.ParseError: XML or text declaration not at start of entity: line 1, column 579

    During handling of the above exception, another exception occurred:

    Traceback (most recent call last):
    File ".\test_all_mock.py", line 20, in <module>
    fetch_response = sqs.receive_message(
        File "C:\Users\dan.small\nswws-public-api-venv\lib\site-packages\botocore\client.py", line 386, in _api_call
    return self._make_api_call(operation_name, kwargs)
    File "C:\Users\dan.small\nswws-public-api-venv\lib\site-packages\botocore\client.py", line 691, in _make_api_call
    http, parsed_response = self._make_request(
        File "C:\Users\dan.small\nswws-public-api-venv\lib\site-packages\botocore\client.py", line 711, in _make_request
    return self._endpoint.make_request(operation_model, request_dict)
    File "C:\Users\dan.small\nswws-public-api-venv\lib\site-packages\botocore\endpoint.py", line 102, in make_request
    return self._send_request(request_dict, operation_model)
    File "C:\Users\dan.small\nswws-public-api-venv\lib\site-packages\botocore\endpoint.py", line 134, in _send_request
    success_response, exception = self._get_response(
        File "C:\Users\dan.small\nswws-public-api-venv\lib\site-packages\botocore\endpoint.py", line 166, in _get_response
    success_response, exception = self._do_get_response(
        File "C:\Users\dan.small\nswws-public-api-venv\lib\site-packages\botocore\endpoint.py", line 217, in _do_get_response
    parsed_response = parser.parse(
        File "C:\Users\dan.small\nswws-public-api-venv\lib\site-packages\botocore\parsers.py", line 245, in parse
    parsed = self._do_parse(response, shape)
    File "C:\Users\dan.small\nswws-public-api-venv\lib\site-packages\botocore\parsers.py", line 551, in _do_parse
    return self._parse_body_as_xml(response, shape, inject_metadata=True)
    File "C:\Users\dan.small\nswws-public-api-venv\lib\site-packages\botocore\parsers.py", line 555, in _parse_body_as_xml
    root = self._parse_xml_string_to_dom(xml_contents)
    File "C:\Users\dan.small\nswws-public-api-venv\lib\site-packages\botocore\parsers.py", line 482, in _parse_xml_string_to_dom
    raise ResponseParserError(
        botocore.parsers.ResponseParserError: Unable to parse response (XML or text declaration not at start of entity: line 1, column 579), invalid XML received. Further retries may succeed: \

    b'<ReceiveMessageResponse><ReceiveMessageResult><Message><MessageId>5f92f101-431a-8a92-0d1c-fce671a53204</MessageId><ReceiptHandle>gaunrwkhvrwdxywiwfrqfqevlmrhdzaeoirlfwnyqwvzcaxyjgcczbklfeidymgvmfnxirgraxedcunnolzugnfxauswrybvisujyylzcaorosgyyynsysqkjqynlhzwfvqwznafcvjftzdammqnlkcufregkmxcjoxvtzibgsifkflkvkgvcrpfv</ReceiptHandle><MD5OfBody>62f853b97bc8b350d6e868c52bf3e5e0</MD5OfBody><Body>NSWSS Atom Feed</Body><MD5OfMessageAttributes>61ca54311f9c73c111b62ff02f407e1f</MD5OfMessageAttributes><MessageAttribute><Name>Payload</Name><Value><DataType>String</DataType><StringValue><?xml version="1.0" encoding="UTF-8"?><feed xmlns="http://www.w3.org/2005/Atom"/></StringValue></Value></MessageAttribute><MessageAttribute><Name>FeedType</Name><Value><DataType>String</DataType><StringValue>Atom</StringValue></Value></MessageAttribute></Message></ReceiveMessageResult><ResponseMetadata><RequestId>KVVLFA4R8TE6MEUUZTZZRS89C0K5IR8U6KMYTTWAR4ZVXUGQ5B0O</RequestId></ResponseMetadata></ReceiveMessageResponse>'
    
What I was expecting was as similar JSON response to using boto3 with AWS SQS.

Using the same code but changing the data to:

	data = {'Payload': {'DataType': 'String', 'StringValue': '<feed xmlns="http://www.w3.org/2005/Atom"/>'}, 'FeedType': {'DataType': 'String', 'StringValue': 'Atom'}}

It prints an empty `Payload` `StringValue`:

    {'Messages': [{'MessageId': 'e98a4d37-ee29-33dd-7c52-9ec2d3588380', 'ReceiptHandle': 'mzlxvlzdxouvkrxbalpvltojegaeourblskjooernlofabbwnrkjsensxgshhwwyqdxxkdoxkeiwnzcwmfwlpdxtkodxhpofmtvrajondaldptrcwqhtzjnedsgtnacnvkfdhqnmrabodmeyklsyipzgnfigtlvhtiypocunisoahpoemrozfknuw', 'MD5OfBody': '62f853b97bc8b350d6e868c52bf3e5e0', 'Body': 'NSWSS Atom Feed', 'MD5OfMessageAttributes': 'f07c25958478170dd7f6ab0e37fbaf19', 'MessageAttributes': {'Payload': {'StringValue': '', 'DataType': 'String'}, 'FeedType': {'StringValue': 'Atom', 'DataType': 'String'}}}], 'ResponseMetadata': {'RequestId': '619FSGJZL1R41AIAPURL2XB9XFEPT9NTXTHUW71NK1G73ROK812K', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'amazon.com', 'x-amzn-requestid': '619FSGJZL1R41AIAPURL2XB9XFEPT9NTXTHUW71NK1G73ROK812K', 'x-amz-crc32': '3186845082'}, 'RetryAttempts': 0}}
    
What I was expecting was as similar JSON response to using boto3 with AWS SQS i.e. not an empty `Payload` `StringValue`.

Using (all installed via pip):
moto==2.2.7
boto3==1.18.44
botocore==1.21.44



Discussion:
Thanks for raising this and providing the test case @sheridansmall - that's very helpful. Looks like we're not properly encoding our payload.

Resolving patch:
diff --git a/moto/sqs/responses.py b/moto/sqs/responses.py
--- a/moto/sqs/responses.py
+++ b/moto/sqs/responses.py
@@ -648,7 +648,7 @@ def list_queue_tags(self):
                 {% if 'Binary' in value.data_type %}
                 <BinaryValue>{{ value.binary_value }}</BinaryValue>
                 {% else %}
-                <StringValue>{{ value.string_value }}</StringValue>
+                <StringValue><![CDATA[{{ value.string_value }}]]></StringValue>
                 {% endif %}
               </Value>
             </MessageAttribute>


Test patch:
diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py
--- a/tests/test_sqs/test_sqs.py
+++ b/tests/test_sqs/test_sqs.py
@@ -916,6 +916,28 @@ def test_send_receive_message_with_attributes_with_labels():
     )
 
 
+@mock_sqs
+def test_receive_message_with_xml_content():
+    sqs = boto3.client("sqs", region_name="eu-west-2")
+    queue_url = sqs.create_queue(QueueName="test-queue")["QueueUrl"]
+    original_payload = '<?xml version="1.0" encoding="UTF-8"?><feed xmlns="http://www.w3.org/2005/Atom"/>'
+    data = {"Payload": {"DataType": "String", "StringValue": original_payload}}
+
+    sqs.send_message(
+        QueueUrl=queue_url, MessageBody="NSWSS Atom Feed", MessageAttributes=data
+    )
+
+    messages = sqs.receive_message(
+        QueueUrl=queue_url,
+        MessageAttributeNames=("Payload",),
+        MaxNumberOfMessages=1,
+        VisibilityTimeout=0,
+    )["Messages"]
+
+    attr = messages[0]["MessageAttributes"]["Payload"]["StringValue"]
+    attr.should.equal(original_payload)
+
+
 @mock_sqs
 def test_change_message_visibility_than_permitted():
     if settings.TEST_SERVER_MODE: