Repository: getmoto/moto

Issue:
mock_sqs: Unable to retrieve multiple messages in a single request
I am trying to receive multiple messages per request but I only see one. If I run the code without mocking, I see both messages sent below in the code. I tried introducing time.sleep(3) in between sending messages and receiving the message but I end up with the same results.

**Versions**
boto==2.49.0
boto3==1.13.1
botocore==1.16.1
moto==1.3.14

**Code**
```
@mock_sqs
def test_create_fifo_queue(aws_credentials):
    session = boto3.Session(profile_name='testing')
    sqs = session.client("sqs", region_name="us-east-1")
    today = datetime.today().strftime('%Y-%m-%d')
    queue_name = f"{os.environ['AWS_RETRY_Q']}.fifo"  

    sqs.create_queue(QueueName=queue_name,
                     Attributes={"FifoQueue": "true"})

    response = sqs.get_queue_url(QueueName=queue_name)
    queue_url = response['QueueUrl']
    print(queue_url)

    sqs.send_message(QueueUrl=queue_url,
                    MessageBody=json.dumps(['A']),
                    MessageAttributes={
                        'Date': {'DataType': 'String',
                                 'StringValue': today},
                        'QueryType': {'DataType': 'String',
                                   'StringValue': "RETRY"}
                    },
                     MessageGroupId='test-queue',
                     MessageDeduplicationId=f'{today}-A')

    sqs.send_message(QueueUrl=queue_url,
                    MessageBody=json.dumps(['B']),
                    MessageAttributes={
                        'Date': {'DataType': 'String',
                                 'StringValue': today},
                        'QueryType': {'DataType': 'String',
                                   'StringValue': "NEW"}
                    },
                     MessageGroupId='test-queue',
                     MessageDeduplicationId=f'{today}-B')

    message_resp = sqs.receive_message(QueueUrl=queue_url,
                                       AttributeNames=['All'],
                                       MessageAttributeNames=['All'],
                                       VisibilityTimeout=20,
                                       MaxNumberOfMessages=10,
                                       WaitTimeSeconds=5
                                       )  #this works on production, receive 2 messages
    print('message1', message_resp['Messages'])
    assert len(message_resp['Messages'])==2
```
**traceback**
AssertionError: assert 1 == 2
E        +  where 1 = len([{'Attributes': {'ApproximateFirstReceiveTimestamp': '1589837266039', 'ApproximateReceiveCount': '1', 'MessageDeduplic...', 'MD5OfBody': '5c24fc1df56d764b550ceae1b9319125', 'MD5OfMessageAttributes': '392230c5ed0aa0660ddc3fd110bdd305', ...}])

test_sqs.py:63: AssertionError



Discussion:

Resolving patch:
diff --git a/moto/sqs/models.py b/moto/sqs/models.py
--- a/moto/sqs/models.py
+++ b/moto/sqs/models.py
@@ -6,6 +6,7 @@
 import re
 import six
 import struct
+from copy import deepcopy
 from xml.sax.saxutils import escape
 
 from boto3 import Session
@@ -101,7 +102,6 @@ def utf8(str):
             if data_type == "String" or data_type == "Number":
                 value = attr["string_value"]
             elif data_type == "Binary":
-                print(data_type, attr["binary_value"], type(attr["binary_value"]))
                 value = base64.b64decode(attr["binary_value"])
             else:
                 print(
@@ -722,6 +722,7 @@ def receive_messages(
         previous_result_count = len(result)
 
         polling_end = unix_time() + wait_seconds_timeout
+        currently_pending_groups = deepcopy(queue.pending_message_groups)
 
         # queue.messages only contains visible messages
         while True:
@@ -739,11 +740,11 @@ def receive_messages(
                     # The message is pending but is visible again, so the
                     # consumer must have timed out.
                     queue.pending_messages.remove(message)
+                    currently_pending_groups = deepcopy(queue.pending_message_groups)
 
                 if message.group_id and queue.fifo_queue:
-                    if message.group_id in queue.pending_message_groups:
-                        # There is already one active message with the same
-                        # group, so we cannot deliver this one.
+                    if message.group_id in currently_pending_groups:
+                        # A previous call is still processing messages in this group, so we cannot deliver this one.
                         continue
 
                 queue.pending_messages.add(message)
diff --git a/moto/sqs/responses.py b/moto/sqs/responses.py
--- a/moto/sqs/responses.py
+++ b/moto/sqs/responses.py
@@ -232,6 +232,14 @@ def send_message(self):
 
         queue_name = self._get_queue_name()
 
+        if not message_group_id:
+            queue = self.sqs_backend.get_queue(queue_name)
+            if queue.attributes.get("FifoQueue", False):
+                return self._error(
+                    "MissingParameter",
+                    "The request must contain the parameter MessageGroupId.",
+                )
+
         message = self.sqs_backend.send_message(
             queue_name,
             message,


Test patch:
diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py
--- a/tests/test_sqs/test_sqs.py
+++ b/tests/test_sqs/test_sqs.py
@@ -1164,7 +1164,7 @@ def test_send_message_batch_with_empty_list():
 
 @mock_sqs
 def test_batch_change_message_visibility():
-    if os.environ.get("TEST_SERVER_MODE", "false").lower() == "true":
+    if settings.TEST_SERVER_MODE:
         raise SkipTest("Cant manipulate time in server mode")
 
     with freeze_time("2015-01-01 12:00:00"):
@@ -1174,9 +1174,15 @@ def test_batch_change_message_visibility():
         )
         queue_url = resp["QueueUrl"]
 
-        sqs.send_message(QueueUrl=queue_url, MessageBody="msg1")
-        sqs.send_message(QueueUrl=queue_url, MessageBody="msg2")
-        sqs.send_message(QueueUrl=queue_url, MessageBody="msg3")
+        sqs.send_message(
+            QueueUrl=queue_url, MessageBody="msg1", MessageGroupId="group1"
+        )
+        sqs.send_message(
+            QueueUrl=queue_url, MessageBody="msg2", MessageGroupId="group2"
+        )
+        sqs.send_message(
+            QueueUrl=queue_url, MessageBody="msg3", MessageGroupId="group3"
+        )
 
     with freeze_time("2015-01-01 12:01:00"):
         receive_resp = sqs.receive_message(QueueUrl=queue_url, MaxNumberOfMessages=2)
@@ -1529,7 +1535,7 @@ def test_create_fifo_queue_with_dlq():
 
 @mock_sqs
 def test_queue_with_dlq():
-    if os.environ.get("TEST_SERVER_MODE", "false").lower() == "true":
+    if settings.TEST_SERVER_MODE:
         raise SkipTest("Cant manipulate time in server mode")
 
     sqs = boto3.client("sqs", region_name="us-east-1")
@@ -1554,8 +1560,12 @@ def test_queue_with_dlq():
         )
         queue_url2 = resp["QueueUrl"]
 
-        sqs.send_message(QueueUrl=queue_url2, MessageBody="msg1")
-        sqs.send_message(QueueUrl=queue_url2, MessageBody="msg2")
+        sqs.send_message(
+            QueueUrl=queue_url2, MessageBody="msg1", MessageGroupId="group"
+        )
+        sqs.send_message(
+            QueueUrl=queue_url2, MessageBody="msg2", MessageGroupId="group"
+        )
 
     with freeze_time("2015-01-01 13:00:00"):
         resp = sqs.receive_message(
@@ -1686,20 +1696,24 @@ def test_receive_messages_with_message_group_id():
     queue.set_attributes(Attributes={"VisibilityTimeout": "3600"})
     queue.send_message(MessageBody="message-1", MessageGroupId="group")
     queue.send_message(MessageBody="message-2", MessageGroupId="group")
+    queue.send_message(MessageBody="message-3", MessageGroupId="group")
+    queue.send_message(MessageBody="separate-message", MessageGroupId="anothergroup")
 
-    messages = queue.receive_messages()
-    messages.should.have.length_of(1)
-    message = messages[0]
+    messages = queue.receive_messages(MaxNumberOfMessages=2)
+    messages.should.have.length_of(2)
+    messages[0].attributes["MessageGroupId"].should.equal("group")
 
-    # received message is not deleted!
-
-    messages = queue.receive_messages(WaitTimeSeconds=0)
-    messages.should.have.length_of(0)
+    # Different client can not 'see' messages from the group until they are processed
+    messages_for_client_2 = queue.receive_messages(WaitTimeSeconds=0)
+    messages_for_client_2.should.have.length_of(1)
+    messages_for_client_2[0].body.should.equal("separate-message")
 
     # message is now processed, next one should be available
-    message.delete()
+    for message in messages:
+        message.delete()
     messages = queue.receive_messages()
     messages.should.have.length_of(1)
+    messages[0].body.should.equal("message-3")
 
 
 @mock_sqs
@@ -1730,7 +1744,7 @@ def test_receive_messages_with_message_group_id_on_requeue():
 
 @mock_sqs
 def test_receive_messages_with_message_group_id_on_visibility_timeout():
-    if os.environ.get("TEST_SERVER_MODE", "false").lower() == "true":
+    if settings.TEST_SERVER_MODE:
         raise SkipTest("Cant manipulate time in server mode")
 
     with freeze_time("2015-01-01 12:00:00"):
@@ -1746,12 +1760,12 @@ def test_receive_messages_with_message_group_id_on_visibility_timeout():
         messages.should.have.length_of(1)
         message = messages[0]
 
-        # received message is not deleted!
-
-        messages = queue.receive_messages(WaitTimeSeconds=0)
-        messages.should.have.length_of(0)
+        # received message is not processed yet
+        messages_for_second_client = queue.receive_messages(WaitTimeSeconds=0)
+        messages_for_second_client.should.have.length_of(0)
 
-        message.change_visibility(VisibilityTimeout=10)
+        for message in messages:
+            message.change_visibility(VisibilityTimeout=10)
 
     with freeze_time("2015-01-01 12:00:05"):
         # no timeout yet
@@ -1794,3 +1808,20 @@ def test_list_queues_limits_to_1000_queues():
     list(resource.queues.filter(QueueNamePrefix="test-queue")).should.have.length_of(
         1000
     )
+
+
+@mock_sqs
+def test_send_messages_to_fifo_without_message_group_id():
+    sqs = boto3.resource("sqs", region_name="eu-west-3")
+    queue = sqs.create_queue(
+        QueueName="blah.fifo",
+        Attributes={"FifoQueue": "true", "ContentBasedDeduplication": "true"},
+    )
+
+    with assert_raises(Exception) as e:
+        queue.send_message(MessageBody="message-1")
+    ex = e.exception
+    ex.response["Error"]["Code"].should.equal("MissingParameter")
+    ex.response["Error"]["Message"].should.equal(
+        "The request must contain the parameter MessageGroupId."
+    )