Repository: localstack/localstack

Issue:
feature request: support for reporting partial batch failure for a SQS batch
### Is there an existing issue for this?

- [X] I have searched the existing issues

### Feature description

AWS lambda has [support for reporting partial batch failure for a SQS batch](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#services-sqs-batchfailurereporting). We're currently using it for our lambda functions on our [serverless.yml](https://www.serverless.com/blog/improved-sqs-batch-error-handling-with-aws-lambda), and would like to test the configuration locally. It'd be great to have it supported on LocalStack.


### 🧑‍💻 Implementa

_No response_

### Anything else?

![Screenshot 2022-01-19 at 6 00 31 PM](https://user-images.githubusercontent.com/39733588/150461615-949ec0b5-564b-4c0c-8a51-0047a7f0faef.png)


```
            "FunctionResponseTypes": [
                "ReportBatchItemFailures"
            ]
```
is missing from localstack's event source mapping despite setting it


Discussion:
We would also :heart: to see this feature coming to localstack
Hi! 

I've this same need.

Hi,

I have the same issue!

Should be a solution anytime soon?

Resolving patch:
diff --git a/localstack/services/awslambda/event_source_listeners/sqs_event_source_listener.py b/localstack/services/awslambda/event_source_listeners/sqs_event_source_listener.py
--- a/localstack/services/awslambda/event_source_listeners/sqs_event_source_listener.py
+++ b/localstack/services/awslambda/event_source_listeners/sqs_event_source_listener.py
@@ -1,3 +1,4 @@
+import json
 import logging
 import time
 from typing import Dict, List
@@ -82,6 +83,10 @@ def _listener_loop(self, *args):
     def _process_messages_for_event_source(self, source, messages):
         lambda_arn = source["FunctionArn"]
         queue_arn = source["EventSourceArn"]
+        # https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#services-sqs-batchfailurereporting
+        report_partial_failures = "ReportBatchItemFailures" in source.get(
+            "FunctionResponseTypes", []
+        )
         region_name = queue_arn.split(":")[3]
         queue_url = aws_stack.sqs_queue_url_for_arn(queue_arn)
         LOG.debug("Sending event from event source %s to Lambda %s", queue_arn, lambda_arn)
@@ -91,22 +96,69 @@ def _process_messages_for_event_source(self, source, messages):
             lambda_arn,
             messages,
             region=region_name,
+            report_partial_failures=report_partial_failures,
         )
         return res
 
-    def _send_event_to_lambda(self, queue_arn, queue_url, lambda_arn, messages, region) -> bool:
-        def delete_messages(result, func_arn, event, error=None, **kwargs):
+    def _send_event_to_lambda(
+        self, queue_arn, queue_url, lambda_arn, messages, region, report_partial_failures=False
+    ) -> bool:
+        records = []
+
+        def delete_messages(result: InvocationResult, func_arn, event, error=None, **kwargs):
             if error:
                 # Skip deleting messages from the queue in case of processing errors. We'll pick them up and retry
                 # next time they become visible in the queue. Redrive policies will be handled automatically by SQS
                 # on the next polling attempt.
+                # Even if ReportBatchItemFailures is set, the entire batch fails if an error is raised.
                 return
 
             region_name = queue_arn.split(":")[3]
             sqs_client = aws_stack.connect_to_service("sqs", region_name=region_name)
-            entries = [
-                {"Id": r["receiptHandle"], "ReceiptHandle": r["receiptHandle"]} for r in records
-            ]
+
+            if report_partial_failures:
+                valid_message_ids = [r["messageId"] for r in records]
+                # collect messages to delete (= the ones that were processed successfully)
+                try:
+                    if messages_to_keep := parse_batch_item_failures(
+                        result, valid_message_ids=valid_message_ids
+                    ):
+                        # unless there is an exception or the parse result is empty, only delete those messages that
+                        # are not part of the partial failure report.
+                        messages_to_delete = [
+                            message_id
+                            for message_id in valid_message_ids
+                            if message_id not in messages_to_keep
+                        ]
+                    else:
+                        # otherwise delete all messages
+                        messages_to_delete = valid_message_ids
+
+                    LOG.debug(
+                        "Lambda partial SQS batch failure report: ok=%s, failed=%s",
+                        messages_to_delete,
+                        messages_to_keep,
+                    )
+                except Exception as e:
+                    LOG.error(
+                        "Error while parsing batchItemFailures from lambda response %s: %s. "
+                        "Treating the batch as complete failure.",
+                        result.result,
+                        e,
+                    )
+                    return
+
+                entries = [
+                    {"Id": r["receiptHandle"], "ReceiptHandle": r["receiptHandle"]}
+                    for r in records
+                    if r["messageId"] in messages_to_delete
+                ]
+
+            else:
+                entries = [
+                    {"Id": r["receiptHandle"], "ReceiptHandle": r["receiptHandle"]} for r in records
+                ]
+
             try:
                 sqs_client.delete_message_batch(QueueUrl=queue_url, Entries=entries)
             except Exception as e:
@@ -115,7 +167,6 @@ def delete_messages(result, func_arn, event, error=None, **kwargs):
                     + "(please check SQS visibility timeout settings): %s - %s" % (entries, e)
                 )
 
-        records = []
         for msg in messages:
             message_attrs = message_attributes_to_lower(msg.get("MessageAttributes"))
             record = {
@@ -149,3 +200,60 @@ def delete_messages(result, func_arn, event, error=None, **kwargs):
             if status_code >= 400:
                 return False
         return True
+
+
+def parse_batch_item_failures(result: InvocationResult, valid_message_ids: List[str]) -> List[str]:
+    """
+    Parses a lambda responses as a partial batch failure response, that looks something like this::
+
+        {
+          "batchItemFailures": [
+                {
+                    "itemIdentifier": "id2"
+                },
+                {
+                    "itemIdentifier": "id4"
+                }
+            ]
+        }
+
+    If the response returns an empty list, then the batch should be considered as a complete success. If an exception
+    is raised, the batch should be considered a complete failure.
+
+    See https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#services-sqs-batchfailurereporting
+
+    :param result: the lambda invocation result
+    :param valid_message_ids: the list of valid message ids in the batch
+    :raises KeyError: if the itemIdentifier value is missing or not in the batch
+    :raises Exception: any other exception related to parsing (e.g., JSON parser error)
+    :return: a list of message IDs that are part of a failure and should not be deleted from the queue
+    """
+    if not result or not result.result:
+        return []
+
+    if isinstance(result.result, dict):
+        partial_batch_failure = result.result
+    else:
+        partial_batch_failure = json.loads(result.result)
+
+    if not partial_batch_failure:
+        return []
+
+    batch_item_failures = partial_batch_failure.get("batchItemFailures")
+
+    if not batch_item_failures:
+        return []
+
+    messages_to_keep = []
+    for item in batch_item_failures:
+        if "itemIdentifier" not in item:
+            raise KeyError(f"missing itemIdentifier in batchItemFailure record {item}")
+
+        item_identifier = item["itemIdentifier"]
+
+        if item_identifier not in valid_message_ids:
+            raise KeyError(f"itemIdentifier '{item_identifier}' not in the batch")
+
+        messages_to_keep.append(item_identifier)
+
+    return messages_to_keep
diff --git a/localstack/services/awslambda/lambda_api.py b/localstack/services/awslambda/lambda_api.py
--- a/localstack/services/awslambda/lambda_api.py
+++ b/localstack/services/awslambda/lambda_api.py
@@ -239,6 +239,9 @@ def build_mapping_obj(data) -> Dict:
     if data.get("DestinationConfig"):
         mapping["DestinationConfig"] = data.get("DestinationConfig")
 
+    if data.get("FunctionResponseTypes"):
+        mapping["FunctionResponseTypes"] = data.get("FunctionResponseTypes")
+
     return mapping
 
 


Test patch:
diff --git a/tests/integration/awslambda/functions/lambda_sqs_batch_item_failure.py b/tests/integration/awslambda/functions/lambda_sqs_batch_item_failure.py
new file mode 100644
--- /dev/null
+++ b/tests/integration/awslambda/functions/lambda_sqs_batch_item_failure.py
@@ -0,0 +1,74 @@
+"""This lambda is used for lambda/sqs integration tests. Since SQS event source mappings don't allow
+DestinationConfigurations that send lambda results to other source (like SQS queues), that can be used to verify
+invocations, this lambda does this manually. You can pass in an event that looks like this::
+
+    {
+        "fail_attempts": 2
+    }
+
+Which will cause the lambda to mark that record as failure twice (comparing the "ApproximateReceiveCount" of the SQS
+event triggering the lambda). The lambda returns a batchItemFailures list that contains every failed record. All
+other records are sent to the DESTINATION_QUEUE_URL as successfully processed.
+
+The lambda understands two env variables:
+* OVERWRITE_RESULT: a string (potentially a json document) that can be used to return custom responses to provoke errors
+* DESTINATION_QUEUE_URL: the queue url to send the event and result to
+"""
+import json
+import os
+
+import boto3
+
+
+def handler(event, context):
+    sqs = create_external_boto_client("sqs")
+
+    print("incoming event:")
+    print(json.dumps(event))
+
+    # this lambda expects inputs from an SQS event source mapping
+    if not event.get("Records"):
+        raise ValueError("no records passed to event")
+
+    batch_item_failures_ids = []
+
+    for record in event["Records"]:
+        message = json.loads(record["body"])
+
+        if message.get("fail_attempts") is None:
+            raise ValueError("no fail_attempts for the event given")
+
+        if message["fail_attempts"] >= int(record["attributes"]["ApproximateReceiveCount"]):
+            batch_item_failures_ids.append(record["messageId"])
+
+    result = {
+        "batchItemFailures": [
+            {"itemIdentifier": message_id} for message_id in batch_item_failures_ids
+        ]
+    }
+
+    if os.environ.get("OVERWRITE_RESULT") is not None:
+        # try to parse the overwrite result as json
+        result = os.environ.get("OVERWRITE_RESULT")
+        try:
+            result = json.loads(result)
+        except Exception:
+            pass
+
+    destination_queue_url = os.environ.get("DESTINATION_QUEUE_URL")
+    if destination_queue_url:
+        sqs.send_message(
+            QueueUrl=destination_queue_url,
+            MessageBody=json.dumps({"event": event, "result": result}),
+        )
+
+    return result
+
+
+def create_external_boto_client(service):
+    endpoint_url = None
+    if os.environ.get("LOCALSTACK_HOSTNAME"):
+        endpoint_url = (
+            f"http://{os.environ['LOCALSTACK_HOSTNAME']}:{os.environ.get('EDGE_PORT', 4566)}"
+        )
+    return boto3.client(service, endpoint_url=endpoint_url)
diff --git a/tests/integration/awslambda/test_lambda_sqs_integration.py b/tests/integration/awslambda/test_lambda_sqs_integration.py
--- a/tests/integration/awslambda/test_lambda_sqs_integration.py
+++ b/tests/integration/awslambda/test_lambda_sqs_integration.py
@@ -16,6 +16,9 @@
 
 THIS_FOLDER = os.path.dirname(os.path.realpath(__file__))
 LAMBDA_SQS_INTEGRATION_FILE = os.path.join(THIS_FOLDER, "functions", "lambda_sqs_integration.py")
+LAMBDA_SQS_BATCH_ITEM_FAILURE_FILE = os.path.join(
+    THIS_FOLDER, "functions", "lambda_sqs_batch_item_failure.py"
+)
 
 
 def _await_event_source_mapping_enabled(lambda_client, uuid, retries=30):
@@ -25,12 +28,27 @@ def assert_mapping_enabled():
     retry(assert_mapping_enabled, sleep_before=2, retries=retries)
 
 
+def _await_queue_size(sqs_client, queue_url: str, qsize: int, retries=10, sleep=1):
+    # wait for all items to appear in the queue
+    def _verify_event_queue_size():
+        attr = "ApproximateNumberOfMessages"
+        _approx = int(
+            sqs_client.get_queue_attributes(QueueUrl=queue_url, AttributeNames=[attr])[
+                "Attributes"
+            ][attr]
+        )
+        assert _approx >= qsize
+
+    retry(_verify_event_queue_size, retries=retries, sleep=sleep)
+
+
 @pytest.fixture(autouse=True)
 def _snapshot_transformers(snapshot):
     # manual transformers since we are passing SQS attributes through lambdas and back again
     snapshot.add_transformer(snapshot.transform.key_value("QueueUrl"))
     snapshot.add_transformer(snapshot.transform.key_value("ReceiptHandle"))
     snapshot.add_transformer(snapshot.transform.key_value("SenderId", reference_replacement=False))
+    snapshot.add_transformer(snapshot.transform.key_value("SequenceNumber"))
     snapshot.add_transformer(snapshot.transform.resource_name())
     # body contains dynamic attributes so md5 hash changes
     snapshot.add_transformer(snapshot.transform.key_value("MD5OfBody"))
@@ -245,7 +263,6 @@ def test_redrive_policy_with_failing_lambda(
     snapshot.match("dlq_response", dlq_response)
 
 
-@pytest.mark.aws_validated
 def test_sqs_queue_as_lambda_dead_letter_queue(
     sqs_client,
     lambda_client,
@@ -307,3 +324,440 @@ def receive_dlq():
     messages = retry(receive_dlq, retries=retries, sleep=3)
 
     snapshot.match("messages", messages)
+
+
+@pytest.mark.skip_snapshot_verify(
+    paths=[
+        # FIXME: we don't seem to be returning SQS FIFO sequence numbers correctly
+        "$..SequenceNumber",
+        # no idea why this one fails
+        "$..receiptHandle",
+        # matching these attributes doesn't work well because of the dynamic nature of messages
+        "$..md5OfBody",
+        "$..MD5OfMessageBody",
+        # FIXME: this is most of the event source mapping unfortunately
+        "$..create_event_source_mapping.ParallelizationFactor",
+        "$..create_event_source_mapping.LastProcessingResult",
+        "$..create_event_source_mapping.Topics",
+        "$..create_event_source_mapping.MaximumRetryAttempts",
+        "$..create_event_source_mapping.MaximumBatchingWindowInSeconds",
+        "$..create_event_source_mapping.FunctionResponseTypes",
+        "$..create_event_source_mapping.StartingPosition",
+        "$..create_event_source_mapping.StateTransitionReason",
+        "$..create_event_source_mapping.State",
+        "$..create_event_source_mapping.ResponseMetadata",
+    ]
+)
+def test_report_batch_item_failures(
+    create_lambda_function,
+    lambda_client,
+    sqs_client,
+    sqs_create_queue,
+    sqs_queue_arn,
+    lambda_su_role,
+    snapshot,
+    cleanups,
+):
+    """This test verifies the SQS Lambda integration feature Reporting batch item failures
+    redrive policy, and the lambda is invoked the correct number of times. The test retries twice and the event
+    source mapping should then automatically move the message to the DQL, but not earlier (see
+    https://github.com/localstack/localstack/issues/5283)"""
+
+    # create queue used in the lambda to send invocation results to (to verify lambda was invoked)
+    destination_queue_name = f"destination-queue-{short_uid()}"
+    destination_url = sqs_create_queue(QueueName=destination_queue_name)
+    snapshot.match(
+        "get_destination_queue_url", sqs_client.get_queue_url(QueueName=destination_queue_name)
+    )
+
+    # timeout in seconds, used for both the lambda and the queue visibility timeout.
+    # increase to 10 if testing against AWS fails.
+    retry_timeout = 8
+    retries = 2
+
+    # set up lambda function
+    function_name = f"failing-lambda-{short_uid()}"
+    create_lambda_function(
+        func_name=function_name,
+        handler_file=LAMBDA_SQS_BATCH_ITEM_FAILURE_FILE,
+        runtime=LAMBDA_RUNTIME_PYTHON38,
+        role=lambda_su_role,
+        timeout=retry_timeout,  # timeout needs to be <= than visibility timeout
+        envvars={"DESTINATION_QUEUE_URL": destination_url},
+    )
+
+    # create dlq for event source queue
+    event_dlq_url = sqs_create_queue(
+        QueueName=f"event-dlq-{short_uid()}.fifo", Attributes={"FifoQueue": "true"}
+    )
+    event_dlq_arn = sqs_queue_arn(event_dlq_url)
+
+    # create event source queue
+    # we use a FIFO queue to be sure the lambda is invoked in a deterministic way
+    event_source_url = sqs_create_queue(
+        QueueName=f"source-queue-{short_uid()}.fifo",
+        Attributes={
+            "FifoQueue": "true",
+            # the visibility timeout is implicitly also the time between retries
+            "VisibilityTimeout": str(retry_timeout),
+            "RedrivePolicy": json.dumps(
+                {"deadLetterTargetArn": event_dlq_arn, "maxReceiveCount": retries}
+            ),
+        },
+    )
+    event_source_arn = sqs_queue_arn(event_source_url)
+
+    # put a batch in the queue. the event format is expected by the lambda_sqs_batch_item_failure.py lambda.
+    # we add the batch before the event_source_mapping to be sure that the entire batch is sent to the first invocation.
+    # message 1 succeeds immediately
+    # message 2 and 3 succeeds after one retry
+    # message 4 fails after 2 retries and lands in the DLQ
+    response = sqs_client.send_message_batch(
+        QueueUrl=event_source_url,
+        Entries=[
+            {
+                "Id": "message-1",
+                "MessageBody": json.dumps({"message": 1, "fail_attempts": 0}),
+                "MessageGroupId": "1",
+                "MessageDeduplicationId": "dedup-1",
+            },
+            {
+                "Id": "message-2",
+                "MessageBody": json.dumps({"message": 2, "fail_attempts": 1}),
+                "MessageGroupId": "1",
+                "MessageDeduplicationId": "dedup-2",
+            },
+            {
+                "Id": "message-3",
+                "MessageBody": json.dumps({"message": 3, "fail_attempts": 1}),
+                "MessageGroupId": "1",
+                "MessageDeduplicationId": "dedup-3",
+            },
+            {
+                "Id": "message-4",
+                "MessageBody": json.dumps({"message": 4, "fail_attempts": retries}),
+                "MessageGroupId": "1",
+                "MessageDeduplicationId": "dedup-4",
+            },
+        ],
+    )
+    # sort so snapshotting works
+    response["Successful"].sort(key=lambda r: r["Id"])
+    snapshot.match("send_message_batch", response)
+
+    # wait for all items to appear in the queue
+    _await_queue_size(sqs_client, event_source_url, qsize=4, retries=30)
+
+    # wire everything with the event source mapping
+    response = lambda_client.create_event_source_mapping(
+        EventSourceArn=event_source_arn,
+        FunctionName=function_name,
+        BatchSize=10,
+        MaximumBatchingWindowInSeconds=0,
+        FunctionResponseTypes=["ReportBatchItemFailures"],
+    )
+    snapshot.match("create_event_source_mapping", response)
+    mapping_uuid = response["UUID"]
+    cleanups.append(lambda: lambda_client.delete_event_source_mapping(UUID=mapping_uuid))
+    _await_event_source_mapping_enabled(lambda_client, mapping_uuid)
+
+    # now wait for the first invocation result which is expected to have processed message 1 we wait half the retry
+    # interval to wait long enough for the message to appear, but short enough to check that the DLQ is empty after
+    # the first attempt.
+    first_invocation = sqs_client.receive_message(
+        QueueUrl=destination_url, WaitTimeSeconds=int(retry_timeout / 2), MaxNumberOfMessages=1
+    )
+    assert "Messages" in first_invocation
+    # hack to make snapshot work
+    first_invocation["Messages"][0]["Body"] = json.loads(first_invocation["Messages"][0]["Body"])
+    first_invocation["Messages"][0]["Body"]["event"]["Records"].sort(
+        key=lambda record: json.loads(record["body"])["message"]
+    )
+    snapshot.match("first_invocation", first_invocation)
+
+    # check that the DQL is empty
+    assert "Messages" not in sqs_client.receive_message(QueueUrl=event_dlq_url)
+
+    # now wait for the second invocation result which is expected to have processed message 2 and 3
+    second_invocation = sqs_client.receive_message(
+        QueueUrl=destination_url, WaitTimeSeconds=retry_timeout + 2, MaxNumberOfMessages=1
+    )
+    assert "Messages" in second_invocation
+    # hack to make snapshot work
+    second_invocation["Messages"][0]["Body"] = json.loads(second_invocation["Messages"][0]["Body"])
+    second_invocation["Messages"][0]["Body"]["event"]["Records"].sort(
+        key=lambda record: json.loads(record["body"])["message"]
+    )
+    snapshot.match("second_invocation", second_invocation)
+
+    # here we make sure there's actually not a third attempt, since our retries = 2
+    third_attempt = sqs_client.receive_message(
+        QueueUrl=destination_url, WaitTimeSeconds=1, MaxNumberOfMessages=1
+    )
+    assert "Messages" not in third_attempt
+
+    # now check that message 4 was placed in the DLQ
+    dlq_response = sqs_client.receive_message(QueueUrl=event_dlq_url, WaitTimeSeconds=15)
+    assert "Messages" in dlq_response
+    snapshot.match("dlq_response", dlq_response)
+
+
+def test_report_batch_item_failures_on_lambda_error(
+    create_lambda_function,
+    lambda_client,
+    sqs_client,
+    sqs_create_queue,
+    sqs_queue_arn,
+    lambda_su_role,
+    snapshot,
+    cleanups,
+):
+    # timeout in seconds, used for both the lambda and the queue visibility timeout
+    retry_timeout = 2
+    retries = 2
+
+    # set up lambda function
+    function_name = f"failing-lambda-{short_uid()}"
+    create_lambda_function(
+        func_name=function_name,
+        handler_file=LAMBDA_SQS_INTEGRATION_FILE,
+        runtime=LAMBDA_RUNTIME_PYTHON38,
+        role=lambda_su_role,
+        timeout=retry_timeout,  # timeout needs to be <= than visibility timeout
+    )
+
+    # create dlq for event source queue
+    event_dlq_url = sqs_create_queue(QueueName=f"event-dlq-{short_uid()}")
+    event_dlq_arn = sqs_queue_arn(event_dlq_url)
+
+    # create event source queue
+    event_source_url = sqs_create_queue(
+        QueueName=f"source-queue-{short_uid()}",
+        Attributes={
+            # the visibility timeout is implicitly also the time between retries
+            "VisibilityTimeout": str(retry_timeout),
+            "RedrivePolicy": json.dumps(
+                {"deadLetterTargetArn": event_dlq_arn, "maxReceiveCount": retries}
+            ),
+        },
+    )
+    event_source_arn = sqs_queue_arn(event_source_url)
+
+    # send a batch with a message to the queue that provokes a lambda failure (the lambda tries to parse the body as
+    # JSON, but if it's not a json document, it fails). consequently, the entire batch should be discarded
+    sqs_client.send_message_batch(
+        QueueUrl=event_source_url,
+        Entries=[
+            {
+                "Id": "message-1",
+                "MessageBody": "{not a json body",
+            },
+            {
+                # this one's ok, but will be sent to the DLQ nonetheless because it's part of this bad batch.
+                "Id": "message-2",
+                "MessageBody": json.dumps({"message": 2, "fail_attempts": 0}),
+            },
+        ],
+    )
+    _await_queue_size(sqs_client, event_source_url, qsize=2)
+
+    # wire everything with the event source mapping
+    mapping_uuid = lambda_client.create_event_source_mapping(
+        EventSourceArn=event_source_arn,
+        FunctionName=function_name,
+        FunctionResponseTypes=["ReportBatchItemFailures"],
+    )["UUID"]
+    cleanups.append(lambda: lambda_client.delete_event_source_mapping(UUID=mapping_uuid))
+    _await_event_source_mapping_enabled(lambda_client, mapping_uuid)
+
+    # the message should arrive in the DLQ after 2 retries + some time for processing
+
+    messages = []
+
+    def _collect_message():
+        dlq_response = sqs_client.receive_message(QueueUrl=event_dlq_url)
+        messages.extend(dlq_response.get("Messages", []))
+        assert len(messages) >= 2
+
+    # the message should arrive in the DLQ after 2 retries + some time for processing
+    wait_time = retry_timeout * retries
+    retry(_collect_message, retries=10, sleep=1, sleep_before=wait_time)
+
+    snapshot.match("dlq_messages", messages)
+
+
+def test_report_batch_item_failures_invalid_result_json_batch_fails(
+    create_lambda_function,
+    lambda_client,
+    sqs_client,
+    sqs_create_queue,
+    sqs_queue_arn,
+    lambda_su_role,
+    snapshot,
+    cleanups,
+):
+    # create queue used in the lambda to send invocation results to (to verify lambda was invoked)
+    destination_queue_name = f"destination-queue-{short_uid()}"
+    destination_url = sqs_create_queue(QueueName=destination_queue_name)
+    snapshot.match(
+        "get_destination_queue_url", sqs_client.get_queue_url(QueueName=destination_queue_name)
+    )
+
+    # timeout in seconds, used for both the lambda and the queue visibility timeout.
+    # increase to 10 if testing against AWS fails.
+    retry_timeout = 4
+    retries = 2
+
+    # set up lambda function
+    function_name = f"failing-lambda-{short_uid()}"
+    create_lambda_function(
+        func_name=function_name,
+        handler_file=LAMBDA_SQS_BATCH_ITEM_FAILURE_FILE,
+        runtime=LAMBDA_RUNTIME_PYTHON38,
+        role=lambda_su_role,
+        timeout=retry_timeout,  # timeout needs to be <= than visibility timeout
+        envvars={
+            "DESTINATION_QUEUE_URL": destination_url,
+            "OVERWRITE_RESULT": '{"batchItemFailures": [{"foo":"notvalid"}]}',
+        },
+    )
+
+    # create dlq for event source queue
+    event_dlq_url = sqs_create_queue(QueueName=f"event-dlq-{short_uid()}")
+    event_dlq_arn = sqs_queue_arn(event_dlq_url)
+
+    # create event source queue
+    event_source_url = sqs_create_queue(
+        QueueName=f"source-queue-{short_uid()}",
+        Attributes={
+            # the visibility timeout is implicitly also the time between retries
+            "VisibilityTimeout": str(retry_timeout),
+            "RedrivePolicy": json.dumps(
+                {"deadLetterTargetArn": event_dlq_arn, "maxReceiveCount": retries}
+            ),
+        },
+    )
+    event_source_arn = sqs_queue_arn(event_source_url)
+
+    # wire everything with the event source mapping
+    mapping_uuid = lambda_client.create_event_source_mapping(
+        EventSourceArn=event_source_arn,
+        FunctionName=function_name,
+        BatchSize=10,
+        MaximumBatchingWindowInSeconds=0,
+        FunctionResponseTypes=["ReportBatchItemFailures"],
+    )["UUID"]
+    cleanups.append(lambda: lambda_client.delete_event_source_mapping(UUID=mapping_uuid))
+    _await_event_source_mapping_enabled(lambda_client, mapping_uuid)
+
+    # trigger the lambda, the message content doesn't matter because the whole batch should be treated as failure
+    sqs_client.send_message(
+        QueueUrl=event_source_url,
+        MessageBody=json.dumps({"message": 1, "fail_attempts": 0}),
+    )
+
+    # now wait for the first invocation result which is expected to have processed message 1 we wait half the retry
+    # interval to wait long enough for the message to appear, but short enough to check that the DLQ is empty after
+    # the first attempt.
+    first_invocation = sqs_client.receive_message(
+        QueueUrl=destination_url, WaitTimeSeconds=15, MaxNumberOfMessages=1
+    )
+    assert "Messages" in first_invocation
+    snapshot.match("first_invocation", first_invocation)
+
+    # now wait for the second invocation result, which should be a retry of the first
+    second_invocation = sqs_client.receive_message(
+        QueueUrl=destination_url, WaitTimeSeconds=15, MaxNumberOfMessages=1
+    )
+    assert "Messages" in second_invocation
+    # hack to make snapshot work
+    snapshot.match("second_invocation", second_invocation)
+
+    # now check that the messages was placed in the DLQ
+    dlq_response = sqs_client.receive_message(QueueUrl=event_dlq_url, WaitTimeSeconds=15)
+    assert "Messages" in dlq_response
+    snapshot.match("dlq_response", dlq_response)
+
+
+def test_report_batch_item_failures_empty_json_batch_succeeds(
+    create_lambda_function,
+    lambda_client,
+    sqs_client,
+    sqs_create_queue,
+    sqs_queue_arn,
+    lambda_su_role,
+    snapshot,
+    cleanups,
+):
+    # create queue used in the lambda to send invocation results to (to verify lambda was invoked)
+    destination_queue_name = f"destination-queue-{short_uid()}"
+    destination_url = sqs_create_queue(QueueName=destination_queue_name)
+    snapshot.match(
+        "get_destination_queue_url", sqs_client.get_queue_url(QueueName=destination_queue_name)
+    )
+
+    # timeout in seconds, used for both the lambda and the queue visibility timeout.
+    # increase to 10 if testing against AWS fails.
+    retry_timeout = 4
+    retries = 1
+
+    # set up lambda function
+    function_name = f"failing-lambda-{short_uid()}"
+    create_lambda_function(
+        func_name=function_name,
+        handler_file=LAMBDA_SQS_BATCH_ITEM_FAILURE_FILE,
+        runtime=LAMBDA_RUNTIME_PYTHON38,
+        role=lambda_su_role,
+        timeout=retry_timeout,  # timeout needs to be <= than visibility timeout
+        envvars={"DESTINATION_QUEUE_URL": destination_url, "OVERWRITE_RESULT": "{}"},
+    )
+
+    # create dlq for event source queue
+    event_dlq_url = sqs_create_queue(QueueName=f"event-dlq-{short_uid()}")
+    event_dlq_arn = sqs_queue_arn(event_dlq_url)
+
+    # create event source queue
+    # we use a FIFO queue to be sure the lambda is invoked in a deterministic way
+    event_source_url = sqs_create_queue(
+        QueueName=f"source-queue-{short_uid()}",
+        Attributes={
+            # the visibility timeout is implicitly also the time between retries
+            "VisibilityTimeout": str(retry_timeout),
+            "RedrivePolicy": json.dumps(
+                {"deadLetterTargetArn": event_dlq_arn, "maxReceiveCount": retries}
+            ),
+        },
+    )
+    event_source_arn = sqs_queue_arn(event_source_url)
+
+    # wire everything with the event source mapping
+    mapping_uuid = lambda_client.create_event_source_mapping(
+        EventSourceArn=event_source_arn,
+        FunctionName=function_name,
+        BatchSize=10,
+        MaximumBatchingWindowInSeconds=0,
+        FunctionResponseTypes=["ReportBatchItemFailures"],
+    )["UUID"]
+    cleanups.append(lambda: lambda_client.delete_event_source_mapping(UUID=mapping_uuid))
+    _await_event_source_mapping_enabled(lambda_client, mapping_uuid)
+
+    # trigger the lambda, the message content doesn't matter because the whole batch should be treated as failure
+    sqs_client.send_message(
+        QueueUrl=event_source_url,
+        MessageBody=json.dumps({"message": 1, "fail_attempts": 0}),
+    )
+
+    # now wait for the first invocation result which is expected to have processed message 1 we wait half the retry
+    # interval to wait long enough for the message to appear, but short enough to check that the DLQ is empty after
+    # the first attempt.
+    first_invocation = sqs_client.receive_message(
+        QueueUrl=destination_url, WaitTimeSeconds=15, MaxNumberOfMessages=1
+    )
+    assert "Messages" in first_invocation
+    snapshot.match("first_invocation", first_invocation)
+
+    # now check that the messages was placed in the DLQ
+    dlq_response = sqs_client.receive_message(
+        QueueUrl=event_dlq_url, WaitTimeSeconds=retry_timeout + 1
+    )
+    assert "Messages" not in dlq_response
diff --git a/tests/integration/awslambda/test_lambda_sqs_integration.snapshot.json b/tests/integration/awslambda/test_lambda_sqs_integration.snapshot.json
--- a/tests/integration/awslambda/test_lambda_sqs_integration.snapshot.json
+++ b/tests/integration/awslambda/test_lambda_sqs_integration.snapshot.json
@@ -237,5 +237,478 @@
         }
       }
     }
+  },
+  "tests/integration/awslambda/test_lambda_sqs_integration.py::test_report_batch_item_failures": {
+    "recorded-date": "07-08-2022, 19:18:36",
+    "recorded-content": {
+      "get_destination_queue_url": {
+        "QueueUrl": "<queue-url:1>",
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        }
+      },
+      "send_message_batch": {
+        "Successful": [
+          {
+            "Id": "message-1",
+            "MessageId": "<uuid:1>",
+            "MD5OfMessageBody": "<md5-of-body:1>",
+            "SequenceNumber": "<sequence-number:1>"
+          },
+          {
+            "Id": "message-2",
+            "MessageId": "<uuid:2>",
+            "MD5OfMessageBody": "<md5-of-body:2>",
+            "SequenceNumber": "<sequence-number:2>"
+          },
+          {
+            "Id": "message-3",
+            "MessageId": "<uuid:3>",
+            "MD5OfMessageBody": "<md5-of-body:3>",
+            "SequenceNumber": "<sequence-number:3>"
+          },
+          {
+            "Id": "message-4",
+            "MessageId": "<uuid:4>",
+            "MD5OfMessageBody": "<m-d5-of-body:3>",
+            "SequenceNumber": "<sequence-number:4>"
+          }
+        ],
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        }
+      },
+      "create_event_source_mapping": {
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 202
+        },
+        "UUID": "<uuid:5>",
+        "BatchSize": 10,
+        "MaximumBatchingWindowInSeconds": 0,
+        "EventSourceArn": "arn:aws:sqs:<region>:111111111111:<resource:1>",
+        "FunctionArn": "arn:aws:lambda:<region>:111111111111:function:<resource:2>",
+        "LastModified": "datetime",
+        "State": "Creating",
+        "StateTransitionReason": "USER_INITIATED",
+        "FunctionResponseTypes": [
+          "ReportBatchItemFailures"
+        ]
+      },
+      "first_invocation": {
+        "Messages": [
+          {
+            "MessageId": "<uuid:6>",
+            "ReceiptHandle": "<receipt-handle:1>",
+            "MD5OfBody": "<m-d5-of-body:1>",
+            "Body": {
+              "event": {
+                "Records": [
+                  {
+                    "messageId": "<uuid:1>",
+                    "receiptHandle": "<receipt-handle:4>",
+                    "body": {
+                      "message": 1,
+                      "fail_attempts": 0
+                    },
+                    "attributes": {
+                      "ApproximateReceiveCount": "1",
+                      "SentTimestamp": "timestamp",
+                      "SequenceNumber": "<sequence-number:1>",
+                      "MessageGroupId": "1",
+                      "SenderId": "sender-id",
+                      "MessageDeduplicationId": "dedup-1",
+                      "ApproximateFirstReceiveTimestamp": "timestamp"
+                    },
+                    "messageAttributes": {},
+                    "md5OfBody": "<md5-of-body:1>",
+                    "eventSource": "aws:sqs",
+                    "eventSourceARN": "arn:aws:sqs:<region>:111111111111:<resource:1>",
+                    "awsRegion": "<region>"
+                  },
+                  {
+                    "messageId": "<uuid:2>",
+                    "receiptHandle": "<receipt-handle:5>",
+                    "body": {
+                      "message": 2,
+                      "fail_attempts": 1
+                    },
+                    "attributes": {
+                      "ApproximateReceiveCount": "1",
+                      "SentTimestamp": "timestamp",
+                      "SequenceNumber": "<sequence-number:2>",
+                      "MessageGroupId": "1",
+                      "SenderId": "sender-id",
+                      "MessageDeduplicationId": "dedup-2",
+                      "ApproximateFirstReceiveTimestamp": "timestamp"
+                    },
+                    "messageAttributes": {},
+                    "md5OfBody": "<md5-of-body:2>",
+                    "eventSource": "aws:sqs",
+                    "eventSourceARN": "arn:aws:sqs:<region>:111111111111:<resource:1>",
+                    "awsRegion": "<region>"
+                  },
+                  {
+                    "messageId": "<uuid:3>",
+                    "receiptHandle": "<receipt-handle:6>",
+                    "body": {
+                      "message": 3,
+                      "fail_attempts": 1
+                    },
+                    "attributes": {
+                      "ApproximateReceiveCount": "1",
+                      "SentTimestamp": "timestamp",
+                      "SequenceNumber": "<sequence-number:3>",
+                      "MessageGroupId": "1",
+                      "SenderId": "sender-id",
+                      "MessageDeduplicationId": "dedup-3",
+                      "ApproximateFirstReceiveTimestamp": "timestamp"
+                    },
+                    "messageAttributes": {},
+                    "md5OfBody": "<md5-of-body:3>",
+                    "eventSource": "aws:sqs",
+                    "eventSourceARN": "arn:aws:sqs:<region>:111111111111:<resource:1>",
+                    "awsRegion": "<region>"
+                  },
+                  {
+                    "messageId": "<uuid:4>",
+                    "receiptHandle": "<receipt-handle:7>",
+                    "body": {
+                      "message": 4,
+                      "fail_attempts": 2
+                    },
+                    "attributes": {
+                      "ApproximateReceiveCount": "1",
+                      "SentTimestamp": "timestamp",
+                      "SequenceNumber": "<sequence-number:4>",
+                      "MessageGroupId": "1",
+                      "SenderId": "sender-id",
+                      "MessageDeduplicationId": "dedup-4",
+                      "ApproximateFirstReceiveTimestamp": "timestamp"
+                    },
+                    "messageAttributes": {},
+                    "md5OfBody": "<m-d5-of-body:3>",
+                    "eventSource": "aws:sqs",
+                    "eventSourceARN": "arn:aws:sqs:<region>:111111111111:<resource:1>",
+                    "awsRegion": "<region>"
+                  }
+                ]
+              },
+              "result": {
+                "batchItemFailures": [
+                  {
+                    "itemIdentifier": "<uuid:2>"
+                  },
+                  {
+                    "itemIdentifier": "<uuid:3>"
+                  },
+                  {
+                    "itemIdentifier": "<uuid:4>"
+                  }
+                ]
+              }
+            }
+          }
+        ],
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        }
+      },
+      "second_invocation": {
+        "Messages": [
+          {
+            "MessageId": "<uuid:7>",
+            "ReceiptHandle": "<receipt-handle:2>",
+            "MD5OfBody": "<m-d5-of-body:2>",
+            "Body": {
+              "event": {
+                "Records": [
+                  {
+                    "messageId": "<uuid:2>",
+                    "receiptHandle": "<receipt-handle:8>",
+                    "body": {
+                      "message": 2,
+                      "fail_attempts": 1
+                    },
+                    "attributes": {
+                      "ApproximateReceiveCount": "2",
+                      "SentTimestamp": "timestamp",
+                      "SequenceNumber": "<sequence-number:2>",
+                      "MessageGroupId": "1",
+                      "SenderId": "sender-id",
+                      "MessageDeduplicationId": "dedup-2",
+                      "ApproximateFirstReceiveTimestamp": "timestamp"
+                    },
+                    "messageAttributes": {},
+                    "md5OfBody": "<md5-of-body:2>",
+                    "eventSource": "aws:sqs",
+                    "eventSourceARN": "arn:aws:sqs:<region>:111111111111:<resource:1>",
+                    "awsRegion": "<region>"
+                  },
+                  {
+                    "messageId": "<uuid:3>",
+                    "receiptHandle": "<receipt-handle:9>",
+                    "body": {
+                      "message": 3,
+                      "fail_attempts": 1
+                    },
+                    "attributes": {
+                      "ApproximateReceiveCount": "2",
+                      "SentTimestamp": "timestamp",
+                      "SequenceNumber": "<sequence-number:3>",
+                      "MessageGroupId": "1",
+                      "SenderId": "sender-id",
+                      "MessageDeduplicationId": "dedup-3",
+                      "ApproximateFirstReceiveTimestamp": "timestamp"
+                    },
+                    "messageAttributes": {},
+                    "md5OfBody": "<md5-of-body:3>",
+                    "eventSource": "aws:sqs",
+                    "eventSourceARN": "arn:aws:sqs:<region>:111111111111:<resource:1>",
+                    "awsRegion": "<region>"
+                  },
+                  {
+                    "messageId": "<uuid:4>",
+                    "receiptHandle": "<receipt-handle:10>",
+                    "body": {
+                      "message": 4,
+                      "fail_attempts": 2
+                    },
+                    "attributes": {
+                      "ApproximateReceiveCount": "2",
+                      "SentTimestamp": "timestamp",
+                      "SequenceNumber": "<sequence-number:4>",
+                      "MessageGroupId": "1",
+                      "SenderId": "sender-id",
+                      "MessageDeduplicationId": "dedup-4",
+                      "ApproximateFirstReceiveTimestamp": "timestamp"
+                    },
+                    "messageAttributes": {},
+                    "md5OfBody": "<m-d5-of-body:3>",
+                    "eventSource": "aws:sqs",
+                    "eventSourceARN": "arn:aws:sqs:<region>:111111111111:<resource:1>",
+                    "awsRegion": "<region>"
+                  }
+                ]
+              },
+              "result": {
+                "batchItemFailures": [
+                  {
+                    "itemIdentifier": "<uuid:4>"
+                  }
+                ]
+              }
+            }
+          }
+        ],
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        }
+      },
+      "dlq_response": {
+        "Messages": [
+          {
+            "MessageId": "<uuid:4>",
+            "ReceiptHandle": "<receipt-handle:3>",
+            "MD5OfBody": "<m-d5-of-body:3>",
+            "Body": {
+              "message": 4,
+              "fail_attempts": 2
+            }
+          }
+        ],
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        }
+      }
+    }
+  },
+  "tests/integration/awslambda/test_lambda_sqs_integration.py::test_report_batch_item_failures_on_lambda_error": {
+    "recorded-date": "08-08-2022, 21:24:21",
+    "recorded-content": {
+      "dlq_messages": [
+        {
+          "MessageId": "<uuid:1>",
+          "ReceiptHandle": "<receipt-handle:1>",
+          "MD5OfBody": "<m-d5-of-body:1>",
+          "Body": "{not a json body"
+        },
+        {
+          "MessageId": "<uuid:2>",
+          "ReceiptHandle": "<receipt-handle:2>",
+          "MD5OfBody": "<m-d5-of-body:2>",
+          "Body": {
+            "message": 2,
+            "fail_attempts": 0
+          }
+        }
+      ]
+    }
+  },
+  "tests/integration/awslambda/test_lambda_sqs_integration.py::test_report_batch_item_failures_invalid_result_json_batch_fails": {
+    "recorded-date": "12-08-2022, 23:58:04",
+    "recorded-content": {
+      "get_destination_queue_url": {
+        "QueueUrl": "<queue-url:1>",
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        }
+      },
+      "first_invocation": {
+        "Messages": [
+          {
+            "Body": {
+              "event": {
+                "Records": [
+                  {
+                    "messageId": "<uuid:1>",
+                    "receiptHandle": "<receipt-handle:4>",
+                    "body": "{\"message\": 1, \"fail_attempts\": 0}",
+                    "attributes": {
+                      "ApproximateReceiveCount": "1",
+                      "SentTimestamp": "timestamp",
+                      "SenderId": "sender-id",
+                      "ApproximateFirstReceiveTimestamp": "timestamp"
+                    },
+                    "messageAttributes": {},
+                    "md5OfBody": "<m-d5-of-body:3>",
+                    "eventSource": "aws:sqs",
+                    "eventSourceARN": "arn:aws:sqs:<region>:111111111111:<resource:1>",
+                    "awsRegion": "<region>"
+                  }
+                ]
+              },
+              "result": {
+                "batchItemFailures": [
+                  {
+                    "foo": "notvalid"
+                  }
+                ]
+              }
+            },
+            "MD5OfBody": "<m-d5-of-body:1>",
+            "MessageId": "<uuid:2>",
+            "ReceiptHandle": "<receipt-handle:1>"
+          }
+        ],
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        }
+      },
+      "second_invocation": {
+        "Messages": [
+          {
+            "Body": {
+              "event": {
+                "Records": [
+                  {
+                    "messageId": "<uuid:1>",
+                    "receiptHandle": "<receipt-handle:5>",
+                    "body": "{\"message\": 1, \"fail_attempts\": 0}",
+                    "attributes": {
+                      "ApproximateReceiveCount": "2",
+                      "SentTimestamp": "timestamp",
+                      "SenderId": "sender-id",
+                      "ApproximateFirstReceiveTimestamp": "timestamp"
+                    },
+                    "messageAttributes": {},
+                    "md5OfBody": "<m-d5-of-body:3>",
+                    "eventSource": "aws:sqs",
+                    "eventSourceARN": "arn:aws:sqs:<region>:111111111111:<resource:1>",
+                    "awsRegion": "<region>"
+                  }
+                ]
+              },
+              "result": {
+                "batchItemFailures": [
+                  {
+                    "foo": "notvalid"
+                  }
+                ]
+              }
+            },
+            "MD5OfBody": "<m-d5-of-body:2>",
+            "MessageId": "<uuid:3>",
+            "ReceiptHandle": "<receipt-handle:2>"
+          }
+        ],
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        }
+      },
+      "dlq_response": {
+        "Messages": [
+          {
+            "Body": {
+              "message": 1,
+              "fail_attempts": 0
+            },
+            "MD5OfBody": "<m-d5-of-body:3>",
+            "MessageId": "<uuid:1>",
+            "ReceiptHandle": "<receipt-handle:3>"
+          }
+        ],
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        }
+      }
+    }
+  },
+  "tests/integration/awslambda/test_lambda_sqs_integration.py::test_report_batch_item_failures_empty_json_batch_succeeds": {
+    "recorded-date": "13-08-2022, 00:52:53",
+    "recorded-content": {
+      "get_destination_queue_url": {
+        "QueueUrl": "<queue-url:1>",
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        }
+      },
+      "first_invocation": {
+        "Messages": [
+          {
+            "Body": {
+              "event": {
+                "Records": [
+                  {
+                    "messageId": "<uuid:1>",
+                    "receiptHandle": "<receipt-handle:2>",
+                    "body": "{\"message\": 1, \"fail_attempts\": 0}",
+                    "attributes": {
+                      "ApproximateReceiveCount": "1",
+                      "SentTimestamp": "timestamp",
+                      "SenderId": "sender-id",
+                      "ApproximateFirstReceiveTimestamp": "timestamp"
+                    },
+                    "messageAttributes": {},
+                    "md5OfBody": "<md5-of-body:1>",
+                    "eventSource": "aws:sqs",
+                    "eventSourceARN": "arn:aws:sqs:<region>:111111111111:<resource:1>",
+                    "awsRegion": "<region>"
+                  }
+                ]
+              },
+              "result": {}
+            },
+            "MD5OfBody": "<m-d5-of-body:1>",
+            "MessageId": "<uuid:2>",
+            "ReceiptHandle": "<receipt-handle:1>"
+          }
+        ],
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        }
+      }
+    }
   }
 }