Repository: localstack/localstack

Issue:
bug: localstack no longer sends the attribute SequenceNumber for SQS FIFO queues
### Is there an existing issue for this?

- [X] I have searched the existing issues

### Current Behavior

The attribute `SequenceNumber` isn't present in the message for high throughput FIFO queues.

```
docker exec -it localstack awslocal sqs receive-message --queue-url http://localhost:4566/000000000000/my-queue.fifo --attribute-name All
```

```json
{
    "Messages": [
        {
            "MessageId": "...",
            "ReceiptHandle": "...",
            "MD5OfBody": "...",
            "Body": "...",
            "Attributes": {
                "SenderId": "...",
                "SentTimestamp": "...",
                "ApproximateReceiveCount": "...",
                "ApproximateFirstReceiveTimestamp": "...",
                "MessageDeduplicationId": "...",
                "MessageGroupId": "..."
            }
        }
    ]
}
```

### Expected Behavior

Up to version `0.14.2`, the message contains the attribute `SequenceNumber`

```json
{
    "Messages": [
        {
            "MessageId": "...",
            "ReceiptHandle": "...",
            "MD5OfBody": "...",
            "Body": "...",
            "Attributes": {
                "SenderId": "...",
                "SentTimestamp": "...",
                "ApproximateReceiveCount": "...",
                "ApproximateFirstReceiveTimestamp": "...",
                "MessageDeduplicationId": "...",
                "MessageGroupId": "...",
                "SequenceNumber": "09883580580611393766"
            }
        }
    ]
}
```

### How are you starting LocalStack?

With a docker-compose file

### Steps To Reproduce

```
docker exec -it localstack awslocal sns create-topic --name my-topic.fifo --attributes FifoTopic=true

docker exec -it localstack awslocal sqs create-queue --queue-name my-queue.fifo --attributes FifoQueue=true,DeduplicationScope=messageGroup,FifoThroughputLimit=perMessageGroupId

docker exec -it localstack awslocal sns subscribe --topic-arn arn:aws:sns:us-east-1:000000000000:my-topic.fifo --protocol sqs --notification-endpoint http://localhost:4566/000000000000/my-queue.fifo

docker exec -it localstack awslocal sns publish --topic-arn arn:aws:sns:us-east-1:000000000000:my-topic.fifo --message 'my-message' --message-group-id my-group --message-deduplication-id 'my-deduplication-id'

docker exec -it localstack awslocal sqs receive-message --queue-url http://localhost:4566/000000000000/my-queue.fifo --attribute-name All
```

### Environment

```markdown
- OS: Docker on Mac
- LocalStack: 1.0.4
```


### Anything else?

_No response_


Discussion:
Welcome to LocalStack! Thanks for reporting your first issue and our team will be working towards fixing the issue for you or reach out for more background information. We recommend joining our [Slack Community](https://localstack.cloud/contact/) for real-time help and drop a message to LocalStack Pro Support if you are a Pro user! If you are willing to contribute towards fixing this issue, please have a look at our [contributing guidelines](https://github.com/localstack/.github/blob/main/CONTRIBUTING.md) and our [developer guide](https://docs.localstack.cloud/developer-guide/).

Resolving patch:
diff --git a/localstack/services/sqs/provider.py b/localstack/services/sqs/provider.py
--- a/localstack/services/sqs/provider.py
+++ b/localstack/services/sqs/provider.py
@@ -3,11 +3,10 @@
 import hashlib
 import heapq
 import inspect
+import itertools
 import json
 import logging
-import random
 import re
-import string
 import threading
 import time
 from queue import Empty, PriorityQueue
@@ -69,9 +68,12 @@
 from localstack.services.generic_proxy import RegionBackend
 from localstack.services.plugins import ServiceLifecycleHook
 from localstack.utils.aws.aws_stack import parse_arn
-from localstack.utils.common import long_uid, md5, now, start_thread
+from localstack.utils.objects import singleton_factory
 from localstack.utils.run import FuncThread
 from localstack.utils.scheduler import Scheduler
+from localstack.utils.strings import long_uid, md5
+from localstack.utils.threads import start_thread
+from localstack.utils.time import now
 
 LOG = logging.getLogger(__name__)
 
@@ -121,6 +123,14 @@ def __init__(self, message):
         super().__init__("MissingParameter", message, 400, True)
 
 
+@singleton_factory
+def global_message_sequence():
+    # creates a 20-digit number used as the start for the global sequence
+    start = int(time.time()) << 33
+    # itertools.count is thread safe over the GIL since its getAndIncrement operation is a single python bytecode op
+    return itertools.count(start)
+
+
 def generate_message_id():
     return long_uid()
 
@@ -191,6 +201,7 @@ class SqsMessage:
     priority: float
     message_deduplication_id: str
     message_group_id: str
+    sequence_number: str
 
     def __init__(
         self,
@@ -198,6 +209,7 @@ def __init__(
         message: Message,
         message_deduplication_id: str = None,
         message_group_id: str = None,
+        sequence_number: str = None,
     ) -> None:
         self.created = time.time()
         self.message = message
@@ -209,12 +221,15 @@ def __init__(
         self.first_received = None
         self.deleted = False
         self.priority = priority
+        self.sequence_number = sequence_number
 
         attributes = {}
         if message_group_id is not None:
             attributes["MessageGroupId"] = message_group_id
         if message_deduplication_id is not None:
             attributes["MessageDeduplicationId"] = message_deduplication_id
+        if sequence_number is not None:
+            attributes["SequenceNumber"] = sequence_number
 
         if self.message.get("Attributes"):
             self.message["Attributes"].update(attributes)
@@ -473,7 +488,7 @@ def put(
         message_deduplication_id: str = None,
         message_group_id: str = None,
         delay_seconds: int = None,
-    ):
+    ) -> SqsMessage:
         raise NotImplementedError
 
     def get(self, block=True, timeout=None, visibility_timeout: int = None) -> SqsMessage:
@@ -574,9 +589,6 @@ def validate_queue_attributes(self, attributes):
             if k not in valid:
                 raise InvalidAttributeName(f"Unknown Attribute {k}.")
 
-    def generate_sequence_number(self):
-        return None
-
 
 class StandardQueue(SqsQueue):
     def put(
@@ -616,6 +628,8 @@ def put(
         else:
             self.visible.put_nowait(standard_message)
 
+        return standard_message
+
 
 class FifoQueue(SqsQueue):
     deduplication: Dict[str, Dict[str, SqsMessage]]
@@ -672,6 +686,7 @@ def put(
             message,
             message_deduplication_id=dedup_id,
             message_group_id=message_group_id,
+            sequence_number=str(self.next_sequence_number()),
         )
         if visibility_timeout is not None:
             fifo_message.visibility_timeout = visibility_timeout
@@ -705,6 +720,8 @@ def put(
                 self.deduplication[message_group_id] = {}
             self.deduplication[message_group_id][dedup_id] = fifo_message
 
+        return fifo_message
+
     def _assert_queue_name(self, name):
         if not name.endswith(".fifo"):
             raise InvalidParameterValue(
@@ -731,10 +748,8 @@ def validate_queue_attributes(self, attributes):
                 "Invalid value for the parameter FifoQueue. Reason: Modifying queue type is not supported."
             )
 
-    # TODO: If we ever actually need to do something with this number, it needs to be part of
-    #   SQSMessage. This means changing all *put*() signatures to return the saved message.
-    def generate_sequence_number(self):
-        return _create_mock_sequence_number()
+    def next_sequence_number(self):
+        return next(global_message_sequence())
 
 
 class QueueUpdateWorker:
@@ -1121,7 +1136,7 @@ def send_message(
     ) -> SendMessageResult:
         queue = self._resolve_queue(context, queue_url=queue_url)
 
-        message = self._put_message(
+        queue_item = self._put_message(
             queue,
             context,
             message_body,
@@ -1131,11 +1146,12 @@ def send_message(
             message_deduplication_id,
             message_group_id,
         )
+        message = queue_item.message
         return SendMessageResult(
             MessageId=message["MessageId"],
             MD5OfMessageBody=message["MD5OfBody"],
             MD5OfMessageAttributes=message.get("MD5OfMessageAttributes"),
-            SequenceNumber=queue.generate_sequence_number(),
+            SequenceNumber=queue_item.sequence_number,
             MD5OfMessageSystemAttributes=_create_message_attribute_hash(message_system_attributes),
         )
 
@@ -1152,7 +1168,7 @@ def send_message_batch(
         with queue.mutex:
             for entry in entries:
                 try:
-                    message = self._put_message(
+                    queue_item = self._put_message(
                         queue,
                         context,
                         message_body=entry.get("MessageBody"),
@@ -1162,6 +1178,7 @@ def send_message_batch(
                         message_deduplication_id=entry.get("MessageDeduplicationId"),
                         message_group_id=entry.get("MessageGroupId"),
                     )
+                    message = queue_item.message
 
                     successful.append(
                         SendMessageBatchResultEntry(
@@ -1172,7 +1189,7 @@ def send_message_batch(
                             MD5OfMessageSystemAttributes=_create_message_attribute_hash(
                                 message.get("message_system_attributes")
                             ),
-                            SequenceNumber=queue.generate_sequence_number(),
+                            SequenceNumber=queue_item.sequence_number,
                         )
                     )
                 except Exception as e:
@@ -1200,7 +1217,7 @@ def _put_message(
         message_system_attributes: MessageBodySystemAttributeMap = None,
         message_deduplication_id: String = None,
         message_group_id: String = None,
-    ) -> Message:
+    ) -> SqsMessage:
         check_message_content(message_body)
         check_attributes(message_attributes)
         check_attributes(message_system_attributes)
@@ -1216,15 +1233,13 @@ def _put_message(
             MessageAttributes=message_attributes,
         )
 
-        queue.put(
+        return queue.put(
             message=message,
             message_deduplication_id=message_deduplication_id,
             message_group_id=message_group_id,
             delay_seconds=int(delay_seconds) if delay_seconds is not None else None,
         )
 
-        return message
-
     def receive_message(
         self,
         context: RequestContext,
@@ -1516,10 +1531,6 @@ def _assert_batch(self, batch: List):
                 visited.add(entry["Id"])
 
 
-def _create_mock_sequence_number():
-    return "".join(random.choice(string.digits) for _ in range(20))
-
-
 # Method from moto's attribute_md5 of moto/sqs/models.py, separated from the Message Object
 def _create_message_attribute_hash(message_attributes) -> Optional[str]:
     # To avoid the need to check for dict conformity everytime we invoke this function


Test patch:
diff --git a/localstack/testing/snapshots/transformer_utility.py b/localstack/testing/snapshots/transformer_utility.py
--- a/localstack/testing/snapshots/transformer_utility.py
+++ b/localstack/testing/snapshots/transformer_utility.py
@@ -176,6 +176,7 @@ def sqs_api():
         return [
             TransformerUtility.key_value("ReceiptHandle"),
             TransformerUtility.key_value("SenderId"),
+            TransformerUtility.key_value("SequenceNumber"),
             TransformerUtility.jsonpath("$..MessageAttributes.RequestID.StringValue", "request-id"),
             KeyValueBasedTransformer(_resource_name_transformer, "resource"),
         ]
diff --git a/tests/integration/test_sqs.py b/tests/integration/test_sqs.py
--- a/tests/integration/test_sqs.py
+++ b/tests/integration/test_sqs.py
@@ -1079,6 +1079,39 @@ def _collect():
         assert messages[1]["Body"] == "message-2"
         assert messages[2]["Body"] == "message-3"
 
+    @pytest.mark.aws_validated
+    def test_fifo_message_attributes(self, sqs_client, sqs_create_queue, snapshot):
+        snapshot.add_transformer(snapshot.transform.sqs_api())
+
+        queue_url = sqs_create_queue(
+            QueueName=f"queue-{short_uid()}.fifo",
+            Attributes={
+                "FifoQueue": "true",
+                "ContentBasedDeduplication": "true",
+                "VisibilityTimeout": "0",
+                "DeduplicationScope": "messageGroup",
+                "FifoThroughputLimit": "perMessageGroupId",
+            },
+        )
+
+        response = sqs_client.send_message(
+            QueueUrl=queue_url,
+            MessageBody="message-body-1",
+            MessageGroupId="group-1",
+            MessageDeduplicationId="dedup-1",
+        )
+        snapshot.match("send_message", response)
+
+        response = sqs_client.receive_message(
+            QueueUrl=queue_url, AttributeNames=["All"], WaitTimeSeconds=10
+        )
+        snapshot.match("receive_message_0", response)
+        # makes sure that attributes are mutated correctly
+        response = sqs_client.receive_message(
+            QueueUrl=queue_url, AttributeNames=["All"], WaitTimeSeconds=10
+        )
+        snapshot.match("receive_message_1", response)
+
     @pytest.mark.aws_validated
     def test_list_queue_tags(self, sqs_client, sqs_create_queue):
         queue_name = f"queue-{short_uid()}"
@@ -1274,20 +1307,19 @@ def test_redrive_policy_attribute_validity(self, sqs_create_queue, sqs_client, s
             QueueUrl=queue_url, Attributes={"RedrivePolicy": json.dumps(_valid_redrive_policy)}
         )
 
-    @pytest.mark.skip
-    def test_invalid_dead_letter_arn_rejected_before_lookup(self, sqs_create_queue):
-        queue_name = f"queue-{short_uid()}"
+    @pytest.mark.aws_validated
+    @pytest.mark.xfail(reason="behavior not implemented yet")
+    def test_invalid_dead_letter_arn_rejected_before_lookup(self, sqs_create_queue, snapshot):
         dl_dummy_arn = "dummy"
         max_receive_count = 42
         _redrive_policy = {
             "deadLetterTargetArn": dl_dummy_arn,
             "maxReceiveCount": max_receive_count,
         }
-        with pytest.raises(Exception) as e:
-            sqs_create_queue(
-                QueueName=queue_name, Attributes={"RedrivePolicy": json.dumps(_redrive_policy)}
-            )
-        e.match("InvalidParameterValue")
+        with pytest.raises(ClientError) as e:
+            sqs_create_queue(Attributes={"RedrivePolicy": json.dumps(_redrive_policy)})
+
+        snapshot.match("error_response", e.value.response)
 
     @pytest.mark.aws_validated
     def test_set_queue_policy(self, sqs_client, sqs_create_queue):
@@ -1890,6 +1922,35 @@ def test_sequence_number(self, sqs_client, sqs_create_queue):
         send_result = sqs_client.send_message(QueueUrl=queue_url, MessageBody=message_content)
         assert "SequenceNumber" not in send_result
 
+    @pytest.mark.aws_validated
+    def test_fifo_sequence_number_increases(self, sqs_client, sqs_create_queue):
+        fifo_queue_name = f"queue-{short_uid()}.fifo"
+        fifo_queue_url = sqs_create_queue(
+            QueueName=fifo_queue_name, Attributes={"FifoQueue": "true"}
+        )
+
+        send_result_1 = sqs_client.send_message(
+            QueueUrl=fifo_queue_url,
+            MessageBody="message-1",
+            MessageGroupId="group",
+            MessageDeduplicationId="m1",
+        )
+        send_result_2 = sqs_client.send_message(
+            QueueUrl=fifo_queue_url,
+            MessageBody="message-2",
+            MessageGroupId="group",
+            MessageDeduplicationId="m2",
+        )
+        send_result_3 = sqs_client.send_message(
+            QueueUrl=fifo_queue_url,
+            MessageBody="message-3",
+            MessageGroupId="group",
+            MessageDeduplicationId="m3",
+        )
+
+        assert int(send_result_1["SequenceNumber"]) < int(send_result_2["SequenceNumber"])
+        assert int(send_result_2["SequenceNumber"]) < int(send_result_3["SequenceNumber"])
+
     @pytest.mark.aws_validated
     def test_posting_to_fifo_requires_deduplicationid_group_id(self, sqs_client, sqs_create_queue):
         fifo_queue_name = f"queue-{short_uid()}.fifo"
diff --git a/tests/integration/test_sqs.snapshot.json b/tests/integration/test_sqs.snapshot.json
--- a/tests/integration/test_sqs.snapshot.json
+++ b/tests/integration/test_sqs.snapshot.json
@@ -357,5 +357,86 @@
       "create_queue_01": "An error occurred (QueueAlreadyExists) when calling the CreateQueue operation: A queue already exists with the same name and a different value for attribute DelaySeconds",
       "create_queue_02": "An error occurred (QueueAlreadyExists) when calling the CreateQueue operation: A queue already exists with the same name and a different value for attribute DelaySeconds"
     }
+  },
+  "tests/integration/test_sqs.py::TestSqsProvider::test_fifo_message_attributes": {
+    "recorded-date": "21-08-2022, 00:35:54",
+    "recorded-content": {
+      "send_message": {
+        "MD5OfMessageBody": "19c9e282d65f9733bc6b35d50062c7ee",
+        "MessageId": "<uuid:1>",
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        },
+        "SequenceNumber": "<sequence-number:1>"
+      },
+      "receive_message_0": {
+        "Messages": [
+          {
+            "Attributes": {
+              "ApproximateFirstReceiveTimestamp": "timestamp",
+              "ApproximateReceiveCount": "1",
+              "MessageDeduplicationId": "dedup-1",
+              "MessageGroupId": "group-1",
+              "SenderId": "<sender-id:1>",
+              "SentTimestamp": "timestamp",
+              "SequenceNumber": "<sequence-number:1>"
+            },
+            "Body": "message-body-1",
+            "MD5OfBody": "19c9e282d65f9733bc6b35d50062c7ee",
+            "MessageId": "<uuid:1>",
+            "ReceiptHandle": "<receipt-handle:1>"
+          }
+        ],
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        }
+      },
+      "receive_message_1": {
+        "Messages": [
+          {
+            "Attributes": {
+              "ApproximateFirstReceiveTimestamp": "timestamp",
+              "ApproximateReceiveCount": "2",
+              "MessageDeduplicationId": "dedup-1",
+              "MessageGroupId": "group-1",
+              "SenderId": "<sender-id:1>",
+              "SentTimestamp": "timestamp",
+              "SequenceNumber": "<sequence-number:1>"
+            },
+            "Body": "message-body-1",
+            "MD5OfBody": "19c9e282d65f9733bc6b35d50062c7ee",
+            "MessageId": "<uuid:1>",
+            "ReceiptHandle": "<receipt-handle:2>"
+          }
+        ],
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        }
+      }
+    }
+  },
+  "tests/integration/test_sqs.py::TestSqsProvider::test_fifo_sequence_number_increases": {
+    "recorded-date": "21-08-2022, 00:36:25",
+    "recorded-content": {}
+  },
+  "tests/integration/test_sqs.py::TestSqsProvider::test_invalid_dead_letter_arn_rejected_before_lookup": {
+    "recorded-date": "21-08-2022, 00:54:10",
+    "recorded-content": {
+      "error_response": {
+        "Error": {
+          "Code": "InvalidParameterValue",
+          "Detail": null,
+          "Message": "Value {\"deadLetterTargetArn\": \"dummy\", \"maxReceiveCount\": 42} for parameter RedrivePolicy is invalid. Reason: Invalid value for deadLetterTargetArn.",
+          "Type": "Sender"
+        },
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 400
+        }
+      }
+    }
   }
 }