Repository: localstack/localstack

Issue:
bug: SNS Topic Publish to Lambda throws an Exception 
### Is there an existing issue for this?

- [X] I have searched the existing issues

### Current Behavior

I am running an SNS topic that publishes to a lambda function. The localstack code reads a type byte object from the lambda function invocation payload [here](https://github.com/localstack/localstack/blob/master/localstack/services/sns/publisher.py#L187) and then tries to dump the result into json [here](https://github.com/localstack/localstack/blob/master/localstack/services/sns/publisher.py#L863) which results in the following error:
```
sns_pub_0] l.services.sns.publisher   : Unable to run Lambda function on SNS message: Object of type bytes is not JSON serializable
```

### Expected Behavior

The payload bytes should be converted to a string before being dumped to json. The error should not be thrown.

### How are you starting LocalStack?

With a docker-compose file

### Steps To Reproduce

#### How are you starting localstack (e.g., `bin/localstack` command, arguments, or `docker-compose.yml`)

docker-compose up
(Using the standard docker-compose.yaml)

#### Client commands (e.g., AWS SDK code snippet, or sequence of "awslocal" commands)

```
awslocal sns publish --message file://sns/message.json --topic-arn arn:aws:sns:us-east-1:000000000000:<topic_name>
```

### Environment

```markdown
- OS: macOS Ventura 13.2.1
- LocalStack: latest
```


### Anything else?

_No response_


Discussion:
Welcome to LocalStack! Thanks for reporting your first issue and our team will be working towards fixing the issue for you or reach out for more background information. We recommend joining our [Slack Community](https://localstack.cloud/contact/) for real-time help and drop a message to LocalStack Pro Support if you are a Pro user! If you are willing to contribute towards fixing this issue, please have a look at our [contributing guidelines](https://github.com/localstack/.github/blob/main/CONTRIBUTING.md) and our [contributing guide](https://docs.localstack.cloud/contributing/).

Resolving patch:
diff --git a/localstack/services/sns/provider.py b/localstack/services/sns/provider.py
--- a/localstack/services/sns/provider.py
+++ b/localstack/services/sns/provider.py
@@ -3,8 +3,9 @@
 from typing import Dict, List
 
 from botocore.utils import InvalidArnException
+from moto.core.utils import camelcase_to_pascal, underscores_to_camelcase
 from moto.sns import sns_backends
-from moto.sns.models import MAXIMUM_MESSAGE_LENGTH
+from moto.sns.models import MAXIMUM_MESSAGE_LENGTH, SNSBackend
 from moto.sns.utils import is_e164
 
 from localstack.aws.accounts import get_aws_account_id
@@ -109,6 +110,10 @@ def on_after_init(self):
     def get_store() -> SnsStore:
         return sns_stores[get_aws_account_id()][aws_stack.get_region()]
 
+    @staticmethod
+    def _get_moto_backend(context: RequestContext) -> SNSBackend:
+        return sns_backends[context.account_id][context.region]
+
     def add_permission(
         self,
         context: RequestContext,
@@ -258,9 +263,17 @@ def verify_sms_sandbox_phone_number(
     def get_topic_attributes(
         self, context: RequestContext, topic_arn: topicARN
     ) -> GetTopicAttributesResponse:
-        moto_response = call_moto(context)
-        # todo fix some attributes by moto, see snapshot
-        return GetTopicAttributesResponse(**moto_response)
+        moto_response: GetTopicAttributesResponse = call_moto(context)
+        # TODO: fix some attributes by moto, see snapshot
+        # TODO: very hacky way to get the attributes we need instead of a moto patch
+        # would need more work to have the proper format out of moto, maybe extract the model to our store
+        moto_backend = self._get_moto_backend(context)
+        moto_topic_model = moto_backend.topics.get(topic_arn)
+        for attr in vars(moto_topic_model):
+            if "success_feedback" in attr:
+                key = camelcase_to_pascal(underscores_to_camelcase(attr))
+                moto_response["Attributes"][key] = getattr(moto_topic_model, attr)
+        return moto_response
 
     def publish_batch(
         self,
@@ -290,7 +303,7 @@ def publish_batch(
                 raise InvalidParameterException(
                     "Invalid parameter: The MessageGroupId parameter is required for FIFO topics"
                 )
-            moto_sns_backend = sns_backends[context.account_id][context.region]
+            moto_sns_backend = self._get_moto_backend(context)
             if moto_sns_backend.get_topic(arn=topic_arn).content_based_deduplication == "false":
                 if not all(
                     ["MessageDeduplicationId" in entry for entry in publish_batch_request_entries]
@@ -441,7 +454,7 @@ def create_platform_endpoint(
         except CommonServiceException as e:
             # TODO: this was unclear in the old provider, check against aws and moto
             if "DuplicateEndpoint" in e.code:
-                moto_sns_backend = sns_backends[context.account_id][context.region]
+                moto_sns_backend = self._get_moto_backend(context)
                 for e in moto_sns_backend.platform_endpoints.values():
                     if e.token == token:
                         if custom_user_data and custom_user_data != e.custom_user_data:
@@ -544,7 +557,7 @@ def publish(
                 raise InvalidParameterException(
                     "Invalid parameter: The MessageGroupId parameter is required for FIFO topics",
                 )
-            moto_sns_backend = sns_backends[context.account_id][context.region]
+            moto_sns_backend = self._get_moto_backend(context)
             if (
                 moto_sns_backend.get_topic(arn=topic_or_target_arn).content_based_deduplication
                 == "false"
@@ -584,7 +597,7 @@ def publish(
 
         if not phone_number:
             if is_endpoint_publish:
-                moto_sns_backend = sns_backends[context.account_id][context.region]
+                moto_sns_backend = self._get_moto_backend(context)
                 if target_arn not in moto_sns_backend.platform_endpoints:
                     raise InvalidParameterException(
                         "Invalid parameter: TargetArn Reason: No endpoint found for the target arn specified"
diff --git a/localstack/services/sns/publisher.py b/localstack/services/sns/publisher.py
--- a/localstack/services/sns/publisher.py
+++ b/localstack/services/sns/publisher.py
@@ -180,11 +180,13 @@ def _publish(self, context: SnsPublishContext, subscriber: SnsSubscription):
             )
             status_code = inv_result.get("StatusCode")
             payload = inv_result.get("Payload")
-
             if payload:
                 delivery = {
                     "statusCode": status_code,
-                    "providerResponse": payload.read(),
+                    # TODO: normally, this is the lambda RequestId (invocation id)
+                    # but we don't get it from the response, this could change if we set the RequestId when we receive
+                    # the request instead of when sending the response. For now, mock it with a random UUID
+                    "providerResponse": json.dumps({"lambdaRequestId": long_uid()}),
                 }
                 store_delivery_log(context.message, subscriber, success=True, delivery=delivery)
 
@@ -834,13 +836,15 @@ def store_delivery_log(
     message_context: SnsMessage, subscriber: SnsSubscription, success: bool, delivery: dict = None
 ):
     """Store the delivery logs in CloudWatch"""
+    # TODO: this is enabled by default in LocalStack, but not in AWS
+    # TODO: validate format of `delivery` for each Publisher
     log_group_name = subscriber.get("TopicArn", "").replace("arn:aws:", "").replace(":", "/")
     log_stream_name = long_uid()
     invocation_time = int(time.time() * 1000)
 
     delivery = not_none_or(delivery, {})
-    delivery["deliveryId"] = (long_uid(),)
-    delivery["destination"] = (subscriber.get("Endpoint", ""),)
+    delivery["deliveryId"] = long_uid()
+    delivery["destination"] = subscriber.get("Endpoint", "")
     delivery["dwellTimeMs"] = 200
     if not success:
         delivery["attemps"] = 1


Test patch:
diff --git a/tests/integration/test_sns.py b/tests/integration/test_sns.py
--- a/tests/integration/test_sns.py
+++ b/tests/integration/test_sns.py
@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import contextlib
 import json
 import logging
 import queue
@@ -136,6 +137,7 @@ def check_subscription():
 
         sns_client.publish(TopicArn=topic_arn, Subject=subject, Message=message)
 
+        # access events sent by lambda
         events = retry(
             check_expected_lambda_log_events_length,
             retries=10,
@@ -3377,3 +3379,166 @@ def test_filter_policy_for_batch(
         )
         # there should be no messages in this queue
         snapshot.match("messages-with-filter-after-publish-filtered", response_after_publish_filter)
+
+
+class TestSNSPublishDelivery:
+    @pytest.mark.aws_validated
+    @pytest.mark.skip_snapshot_verify(
+        paths=[
+            "$.get-topic-attrs.Attributes.DeliveryPolicy",
+            "$.get-topic-attrs.Attributes.EffectiveDeliveryPolicy",
+            "$.get-topic-attrs.Attributes.Policy.Statement..Action",  # SNS:Receive is added by moto but not returned in AWS
+        ]
+    )
+    def test_delivery_lambda(
+        self,
+        sns_client,
+        sns_create_topic,
+        sns_subscription,
+        lambda_client,
+        lambda_su_role,
+        create_lambda_function,
+        logs_client,
+        iam_client,
+        create_role,
+        create_policy,
+        snapshot,
+    ):
+        snapshot.add_transformers_list(
+            [
+                snapshot.transform.key_value(
+                    "dwellTimeMs", reference_replacement=False, value_replacement="<time-ms>"
+                ),
+                snapshot.transform.key_value("nextBackwardToken"),
+                snapshot.transform.key_value("nextForwardToken"),
+            ]
+        )
+        function_name = f"lambda-function-{short_uid()}"
+        permission_id = f"test-statement-{short_uid()}"
+        subject = "[Subject] Test subject"
+        message = "Hello world."
+        topic_name = f"test-topic-{short_uid()}"
+        topic_arn = sns_create_topic(Name=topic_name)["TopicArn"]
+        parsed_arn = parse_arn(topic_arn)
+        account_id = parsed_arn["account"]
+        region = parsed_arn["region"]
+        role_name = f"SNSSuccessFeedback-{short_uid()}"
+        policy_name = f"SNSSuccessFeedback-policy-{short_uid()}"
+
+        # enable Success Feedback from SNS to be sent to CloudWatch
+        # TODO: this is enabled by default in LS
+        trust_policy = {
+            "Version": "2012-10-17",
+            "Statement": [
+                {
+                    "Effect": "Allow",
+                    "Principal": {"Service": "sns.amazonaws.com"},
+                    "Action": "sts:AssumeRole",
+                }
+            ],
+        }
+        cloudwatch_policy = {
+            "Version": "2012-10-17",
+            "Statement": [
+                {
+                    "Effect": "Allow",
+                    "Action": [
+                        "logs:CreateLogGroup",
+                        "logs:CreateLogStream",
+                        "logs:PutLogEvents",
+                        "logs:PutMetricFilter",
+                        "logs:PutRetentionPolicy",
+                    ],
+                    "Resource": ["*"],
+                }
+            ],
+        }
+
+        role_response = create_role(
+            RoleName=role_name, AssumeRolePolicyDocument=json.dumps(trust_policy)
+        )
+        role_arn = role_response["Role"]["Arn"]
+        policy_arn = create_policy(
+            PolicyName=policy_name, PolicyDocument=json.dumps(cloudwatch_policy)
+        )["Policy"]["Arn"]
+        iam_client.attach_role_policy(RoleName=role_name, PolicyArn=policy_arn)
+        if is_aws_cloud():
+            # wait for the policy to be properly attached
+            time.sleep(20)
+
+        sns_client.set_topic_attributes(
+            TopicArn=topic_arn,
+            AttributeName="LambdaSuccessFeedbackRoleArn",
+            AttributeValue=role_arn,
+        )
+
+        sns_client.set_topic_attributes(
+            TopicArn=topic_arn,
+            AttributeName="LambdaSuccessFeedbackSampleRate",
+            AttributeValue="100",
+        )
+
+        topic_attributes = sns_client.get_topic_attributes(TopicArn=topic_arn)
+        snapshot.match("get-topic-attrs", topic_attributes)
+
+        lambda_creation_response = create_lambda_function(
+            func_name=function_name,
+            handler_file=TEST_LAMBDA_PYTHON_ECHO,
+            runtime=Runtime.python3_7,
+            role=lambda_su_role,
+        )
+        lambda_arn = lambda_creation_response["CreateFunctionResponse"]["FunctionArn"]
+        lambda_client.add_permission(
+            FunctionName=function_name,
+            StatementId=permission_id,
+            Action="lambda:InvokeFunction",
+            Principal="sns.amazonaws.com",
+            SourceArn=topic_arn,
+        )
+
+        subscription = sns_subscription(
+            TopicArn=topic_arn,
+            Protocol="lambda",
+            Endpoint=lambda_arn,
+        )
+
+        def check_subscription():
+            subscription_arn = subscription["SubscriptionArn"]
+            subscription_attrs = sns_client.get_subscription_attributes(
+                SubscriptionArn=subscription_arn
+            )
+            assert subscription_attrs["Attributes"]["PendingConfirmation"] == "false"
+
+        retry(check_subscription, retries=PUBLICATION_RETRIES, sleep=PUBLICATION_TIMEOUT)
+
+        sns_client.publish(TopicArn=topic_arn, Subject=subject, Message=message)
+
+        log_group_name = f"sns/{region}/{account_id}/{topic_name}"
+
+        def get_log_events():
+            log_streams = logs_client.describe_log_streams(logGroupName=log_group_name)[
+                "logStreams"
+            ]
+            assert len(log_streams) == 1
+            log_events = logs_client.get_log_events(
+                logGroupName=log_group_name,
+                logStreamName=log_streams[0]["logStreamName"],
+            )
+            assert len(log_events["events"]) == 1
+            # the default retention is 30 days, so delete the logGroup to clean up AWS
+            with contextlib.suppress(ClientError):
+                logs_client.delete_log_group(logGroupName=log_group_name)
+            return log_events
+
+        sleep_time = 5 if is_aws_cloud() else 0.3
+        events = retry(get_log_events, retries=10, sleep=sleep_time)
+
+        # we need to decode the providerResponse to be able to properly match on the response
+        # test would raise an error anyway if it's not a JSON string
+        msg = json.loads(events["events"][0]["message"])
+        events["events"][0]["message"] = msg
+        events["events"][0]["message"]["delivery"]["providerResponse"] = json.loads(
+            msg["delivery"]["providerResponse"]
+        )
+
+        snapshot.match("delivery-events", events)
diff --git a/tests/integration/test_sns.snapshot.json b/tests/integration/test_sns.snapshot.json
--- a/tests/integration/test_sns.snapshot.json
+++ b/tests/integration/test_sns.snapshot.json
@@ -3208,5 +3208,101 @@
         }
       }
     }
+  },
+  "tests/integration/test_sns.py::TestSNSPublishDelivery::test_delivery_lambda": {
+    "recorded-date": "01-03-2023, 14:56:10",
+    "recorded-content": {
+      "get-topic-attrs": {
+        "Attributes": {
+          "DisplayName": "",
+          "EffectiveDeliveryPolicy": {
+            "http": {
+              "defaultHealthyRetryPolicy": {
+                "minDelayTarget": 20,
+                "maxDelayTarget": 20,
+                "numRetries": 3,
+                "numMaxDelayRetries": 0,
+                "numNoDelayRetries": 0,
+                "numMinDelayRetries": 0,
+                "backoffFunction": "linear"
+              },
+              "disableSubscriptionOverrides": false
+            }
+          },
+          "LambdaSuccessFeedbackRoleArn": "arn:aws:iam::111111111111:role/<resource:1>",
+          "LambdaSuccessFeedbackSampleRate": "100",
+          "Owner": "111111111111",
+          "Policy": {
+            "Version": "2008-10-17",
+            "Id": "__default_policy_ID",
+            "Statement": [
+              {
+                "Sid": "__default_statement_ID",
+                "Effect": "Allow",
+                "Principal": {
+                  "AWS": "*"
+                },
+                "Action": [
+                  "SNS:GetTopicAttributes",
+                  "SNS:SetTopicAttributes",
+                  "SNS:AddPermission",
+                  "SNS:RemovePermission",
+                  "SNS:DeleteTopic",
+                  "SNS:Subscribe",
+                  "SNS:ListSubscriptionsByTopic",
+                  "SNS:Publish"
+                ],
+                "Resource": "arn:aws:sns:<region>:111111111111:<resource:2>",
+                "Condition": {
+                  "StringEquals": {
+                    "AWS:SourceOwner": "111111111111"
+                  }
+                }
+              }
+            ]
+          },
+          "SubscriptionsConfirmed": "0",
+          "SubscriptionsDeleted": "0",
+          "SubscriptionsPending": "0",
+          "TopicArn": "arn:aws:sns:<region>:111111111111:<resource:2>"
+        },
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        }
+      },
+      "delivery-events": {
+        "events": [
+          {
+            "ingestionTime": "timestamp",
+            "message": {
+              "delivery": {
+                "deliveryId": "<uuid:1>",
+                "destination": "arn:aws:lambda:<region>:111111111111:function:<resource:3>",
+                "dwellTimeMs": "<time-ms>",
+                "providerResponse": {
+                  "lambdaRequestId": "<uuid:2>"
+                },
+                "statusCode": 202
+              },
+              "notification": {
+                "messageId": "<uuid:3>",
+                "messageMD5Sum": "764569e58f53ea8b6404f6fa7fc0247f",
+                "timestamp": "timestamp",
+                "topicArn": "arn:aws:sns:<region>:111111111111:<resource:2>"
+              },
+              "status": "SUCCESS"
+            },
+            "timestamp": "timestamp"
+          }
+        ],
+        "nextBackwardToken": "<next-backward-token:1>",
+        "nextForwardToken": "<next-forward-token:1>",
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        }
+      }
+    }
   }
 }