Repository: localstack/localstack

Issue:
SQS provider should raise QueueAlreadyExists only when queue exists with different attributes
In the ASF SQS provider, we always raise a `QueueNameExists` exception if the queue already exists.

https://github.com/localstack/localstack/blob/aba25e4f9542a2cc1ce94b08d50247b8024c3d12/localstack/services/sqs/provider.py#L728-L729
However, [according to the spec](https://github.com/boto/botocore/blob/f960a8d73c12d16b53e4d9eacdf515224acb08f6/botocore/data/sqs/2012-11-05/service-2.json#L1138), "Amazon SQS returns this error only if the request includes attributes whose values differ from those of the existing queue."



Discussion:
adding a quick fix as part of #6016 so it does not raise an exception at all

Resolving patch:
diff --git a/localstack/services/sqs/provider.py b/localstack/services/sqs/provider.py
--- a/localstack/services/sqs/provider.py
+++ b/localstack/services/sqs/provider.py
@@ -51,6 +51,7 @@
     QueueAttributeName,
     QueueDeletedRecently,
     QueueDoesNotExist,
+    QueueNameExists,
     ReceiptHandleIsInvalid,
     ReceiveMessageResult,
     SendMessageBatchRequestEntryList,
@@ -91,6 +92,19 @@
 # see https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_DeleteQueue.html
 RECENTLY_DELETED_TIMEOUT = 60
 
+INTERNAL_QUEUE_ATTRIBUTES = [
+    # these attributes cannot be changed by set_queue_attributes and should
+    # therefore be ignored when comparing queue attributes for create_queue
+    QueueAttributeName.ApproximateNumberOfMessages,
+    QueueAttributeName.ApproximateNumberOfMessagesDelayed,
+    QueueAttributeName.ApproximateNumberOfMessagesNotVisible,
+    QueueAttributeName.ContentBasedDeduplication,
+    QueueAttributeName.CreatedTimestamp,
+    QueueAttributeName.FifoQueue,
+    QueueAttributeName.LastModifiedTimestamp,
+    QueueAttributeName.QueueArn,
+]
+
 
 class InvalidParameterValue(CommonServiceException):
     def __init__(self, message):
@@ -555,7 +569,7 @@ def validate_queue_attributes(self, attributes):
 
         for k in attributes.keys():
             if k not in valid:
-                raise InvalidAttributeName(f"Unknown Attribute {k}")
+                raise InvalidAttributeName(f"Unknown Attribute {k}.")
 
     def generate_sequence_number(self):
         return None
@@ -601,7 +615,6 @@ def put(
 
 
 class FifoQueue(SqsQueue):
-
     deduplication: Dict[str, Dict[str, SqsMessage]]
 
     def __init__(self, name: str, region: str, account_id: str, attributes=None, tags=None) -> None:
@@ -696,7 +709,7 @@ def validate_queue_attributes(self, attributes):
 
         for k in attributes.keys():
             if k not in valid:
-                raise InvalidAttributeName(f"Unknown Attribute {k}")
+                raise InvalidAttributeName(f"Unknown Attribute {k}.")
         # Special Cases
         fifo = attributes.get(QueueAttributeName.FifoQueue)
         if fifo and fifo.lower() != "true":
@@ -919,8 +932,24 @@ def create_queue(
 
         with self._mutex:
             if queue_name in backend.queues:
-                # FIXME #5938: should raise `QueueNameExists` if queue exists with different attributes
                 queue = backend.queues[queue_name]
+
+                if attributes:
+                    # if attributes are set, then we check whether the existing attributes match the passed ones
+                    self._validate_queue_attributes(attributes)
+                    for k, v in attributes.items():
+                        if queue.attributes.get(k) != v:
+                            LOG.debug(
+                                "queue attribute values %s for queue %s do not match %s (existing) != %s (new)",
+                                k,
+                                queue_name,
+                                queue.attributes.get(k),
+                                v,
+                            )
+                            raise QueueNameExists(
+                                f"A queue already exists with the same name and a different value for attribute {k}"
+                            )
+
                 return CreateQueueResult(QueueUrl=queue.url(context))
 
             if config.SQS_DELAY_RECENTLY_DELETED:
@@ -1357,6 +1386,8 @@ def set_queue_attributes(
         queue.validate_queue_attributes(attributes)
 
         for k, v in attributes.items():
+            if k in INTERNAL_QUEUE_ATTRIBUTES:
+                raise InvalidAttributeName(f"Unknown Attribute {k}.")
             queue.attributes[k] = v
 
         # Special cases
@@ -1449,8 +1480,8 @@ def _validate_queue_attributes(self, attributes: QueueAttributeMap):
         valid = [k[1] for k in inspect.getmembers(QueueAttributeName)]
 
         for k in attributes.keys():
-            if k not in valid:
-                raise InvalidAttributeName("Unknown Attribute %s" % k)
+            if k not in valid or k in INTERNAL_QUEUE_ATTRIBUTES:
+                raise InvalidAttributeName(f"Unknown Attribute {k}.")
 
     def _validate_actions(self, actions: ActionNameList):
         service = load_service(service=self.service, version=self.version)


Test patch:
diff --git a/tests/integration/test_sqs.py b/tests/integration/test_sqs.py
--- a/tests/integration/test_sqs.py
+++ b/tests/integration/test_sqs.py
@@ -437,29 +437,84 @@ def _send_message():
         ), f"unexpected number of messages in response {response}"
 
     @pytest.mark.aws_validated
-    @pytest.mark.xfail(reason="see https://github.com/localstack/localstack/issues/5938")
+    def test_create_queue_with_default_attributes_is_idempotent(self, sqs_client, sqs_create_queue):
+        queue_name = f"queue-{short_uid()}"
+        attributes = {
+            "VisibilityTimeout": "69",
+            "ReceiveMessageWaitTimeSeconds": "1",
+        }
+
+        queue_url = sqs_create_queue(QueueName=queue_name, Attributes=attributes)
+        assert sqs_create_queue(QueueName=queue_name) == queue_url
+
+    @pytest.mark.aws_validated
     def test_create_queue_with_different_attributes_raises_exception(
-        self, sqs_client, sqs_create_queue
+        self, sqs_client, sqs_create_queue, snapshot
     ):
         queue_name = f"queue-{short_uid()}"
 
-        sqs_create_queue(QueueName=queue_name)
+        # create queue with ReceiveMessageWaitTimeSeconds=2
+        queue_url = sqs_create_queue(
+            QueueName=queue_name,
+            Attributes={
+                "ReceiveMessageWaitTimeSeconds": "1",
+                "DelaySeconds": "1",
+            },
+        )
 
+        # try to create a queue without attributes works
+        assert queue_url == sqs_create_queue(QueueName=queue_name)
+
+        # try to create a queue with one attribute specified
+        assert queue_url == sqs_create_queue(QueueName=queue_name, Attributes={"DelaySeconds": "1"})
+
+        # try to create a queue with the same name but different ReceiveMessageWaitTimeSeconds value
         with pytest.raises(ClientError) as e:
             sqs_create_queue(
                 QueueName=queue_name,
                 Attributes={
                     "ReceiveMessageWaitTimeSeconds": "1",
+                    "DelaySeconds": "2",
                 },
             )
-        e.match("QueueAlreadyExists")
+        snapshot.match("create_queue_01", e.value)
 
-        assert (
-            e.value.response["Error"]["Message"]
-            == "A queue already exists with the same name and a different value for attribute "
-            "ReceiveMessageWaitTimeSeconds"
+        # update the attribute of the queue
+        sqs_client.set_queue_attributes(QueueUrl=queue_url, Attributes={"DelaySeconds": "2"})
+
+        # try again
+        assert queue_url == sqs_create_queue(
+            QueueName=queue_name,
+            Attributes={
+                "ReceiveMessageWaitTimeSeconds": "1",
+                "DelaySeconds": "2",
+            },
         )
 
+        # try with the original request
+        with pytest.raises(ClientError) as e:
+            sqs_create_queue(
+                QueueName=queue_name,
+                Attributes={
+                    "ReceiveMessageWaitTimeSeconds": "1",
+                    "DelaySeconds": "1",
+                },
+            )
+        snapshot.match("create_queue_02", e.value)
+
+    @pytest.mark.aws_validated
+    def test_create_queue_after_internal_attributes_changes_works(
+        self, sqs_client, sqs_create_queue
+    ):
+        queue_name = f"queue-{short_uid()}"
+
+        queue_url = sqs_create_queue(QueueName=queue_name)
+
+        sqs_client.send_message(QueueUrl=queue_url, MessageBody="foobar-1", DelaySeconds=1)
+        sqs_client.send_message(QueueUrl=queue_url, MessageBody="foobar-2")
+
+        assert queue_url == sqs_create_queue(QueueName=queue_name)
+
     @pytest.mark.aws_validated
     def test_create_and_update_queue_attributes(self, sqs_client, sqs_create_queue, snapshot):
         queue_url = sqs_create_queue(
@@ -485,6 +540,109 @@ def test_create_and_update_queue_attributes(self, sqs_client, sqs_create_queue,
         response = sqs_client.get_queue_attributes(QueueUrl=queue_url, AttributeNames=["All"])
         snapshot.match("get_updated_queue_attributes", response)
 
+    @pytest.mark.aws_validated
+    @pytest.mark.xfail(reason="see https://github.com/localstack/localstack/issues/5938")
+    def test_create_queue_with_default_arguments_works_with_modified_attributes(
+        self, sqs_client, sqs_create_queue
+    ):
+        queue_name = f"queue-{short_uid()}"
+        queue_url = sqs_create_queue(QueueName=queue_name)
+
+        sqs_client.set_queue_attributes(
+            QueueUrl=queue_url,
+            Attributes={
+                "VisibilityTimeout": "2",
+                "ReceiveMessageWaitTimeSeconds": "2",
+            },
+        )
+
+        # original attributes
+        with pytest.raises(ClientError) as e:
+            sqs_create_queue(
+                QueueName=queue_name,
+                Attributes={
+                    "VisibilityTimeout": "1",
+                    "ReceiveMessageWaitTimeSeconds": "1",
+                },
+            )
+        e.match("QueueAlreadyExists")
+
+        # modified attributes
+        assert queue_url == sqs_create_queue(
+            QueueName=queue_name,
+            Attributes={
+                "VisibilityTimeout": "2",
+                "ReceiveMessageWaitTimeSeconds": "2",
+            },
+        )
+
+        # no attributes always works
+        assert queue_url == sqs_create_queue(QueueName=queue_name)
+
+    @pytest.mark.aws_validated
+    @pytest.mark.xfail(reason="see https://github.com/localstack/localstack/issues/5938")
+    def test_create_queue_after_modified_attributes(self, sqs_client, sqs_create_queue):
+        queue_name = f"queue-{short_uid()}"
+        queue_url = sqs_create_queue(
+            QueueName=queue_name,
+            Attributes={
+                "VisibilityTimeout": "1",
+                "ReceiveMessageWaitTimeSeconds": "1",
+            },
+        )
+
+        sqs_client.set_queue_attributes(
+            QueueUrl=queue_url,
+            Attributes={
+                "VisibilityTimeout": "2",
+                "ReceiveMessageWaitTimeSeconds": "2",
+            },
+        )
+
+        # original attributes
+        with pytest.raises(ClientError) as e:
+            sqs_create_queue(
+                QueueName=queue_name,
+                Attributes={
+                    "VisibilityTimeout": "1",
+                    "ReceiveMessageWaitTimeSeconds": "1",
+                },
+            )
+        e.match("QueueAlreadyExists")
+
+        # modified attributes
+        assert queue_url == sqs_create_queue(
+            QueueName=queue_name,
+            Attributes={
+                "VisibilityTimeout": "2",
+                "ReceiveMessageWaitTimeSeconds": "2",
+            },
+        )
+
+        # no attributes always works
+        assert queue_url == sqs_create_queue(QueueName=queue_name)
+
+    @pytest.mark.aws_validated
+    def test_create_queue_after_send(self, sqs_client, sqs_create_queue):
+        # checks that intrinsic queue attributes like "ApproxMessages" does not hinder queue creation
+        queue_name = f"queue-{short_uid()}"
+        queue_url = sqs_create_queue(QueueName=queue_name)
+
+        sqs_client.send_message(QueueUrl=queue_url, MessageBody="foobar")
+        sqs_client.send_message(QueueUrl=queue_url, MessageBody="bared")
+        sqs_client.send_message(QueueUrl=queue_url, MessageBody="baz")
+
+        def _qsize(_url):
+            response = sqs_client.get_queue_attributes(
+                QueueUrl=_url, AttributeNames=["ApproximateNumberOfMessages"]
+            )
+            return int(response["Attributes"]["ApproximateNumberOfMessages"])
+
+        assert poll_condition(lambda: _qsize(queue_url) > 0, timeout=10)
+
+        # we know that the system attribute has changed, now check whether create_queue works
+        assert queue_url == sqs_create_queue(QueueName=queue_name)
+
     @pytest.mark.aws_validated
     def test_send_delay_and_wait_time(self, sqs_client, sqs_queue):
         sqs_client.send_message(QueueUrl=sqs_queue, MessageBody="foobar", DelaySeconds=1)
diff --git a/tests/integration/test_sqs.snapshot.json b/tests/integration/test_sqs.snapshot.json
--- a/tests/integration/test_sqs.snapshot.json
+++ b/tests/integration/test_sqs.snapshot.json
@@ -350,5 +350,12 @@
     "recorded-content": {
       "send_message": "An error occurred (InvalidParameterValue) when calling the SendMessage operation: Value 2 for parameter DelaySeconds is invalid. Reason: The request include parameter that is not valid for this queue type."
     }
+  },
+  "tests/integration/test_sqs.py::TestSqsProvider::test_create_queue_with_different_attributes_raises_exception": {
+    "recorded-date": "04-08-2022, 17:27:47",
+    "recorded-content": {
+      "create_queue_01": "An error occurred (QueueAlreadyExists) when calling the CreateQueue operation: A queue already exists with the same name and a different value for attribute DelaySeconds",
+      "create_queue_02": "An error occurred (QueueAlreadyExists) when calling the CreateQueue operation: A queue already exists with the same name and a different value for attribute DelaySeconds"
+    }
   }
 }