Repository: localstack/localstack

Issue:
No SubscriptionConfirmation message for HTTP(S) endpoints
When you subscribe to a SNS topic a confirmation message should be sent.

For example (taken from amazon docs):
```
{
  "Type" : "SubscriptionConfirmation",
  "MessageId" : "165545c9-2a5c-472c-8df2-7ff2be2b3b1b",
  "Token" : "2336412f37fb687f5d51e6e241d09c805a5a57b30d712f794cc5f6a988666d92768dd60a747ba6f3beb71854e285d6ad02428b09ceece29417f1f02d609c582afbacc99c583a916b9981dd2728f4ae6fdb82efd087cc3b7849e05798d2d2785c03b0879594eeac82c01f235d0e717736",
  "TopicArn" : "arn:aws:sns:us-west-2:123456789012:MyTopic",
  "Message" : "You have chosen to subscribe to the topic arn:aws:sns:us-west-2:123456789012:MyTopic.\nTo confirm the subscription, visit the SubscribeURL included in this message.",
  "SubscribeURL" : "https://sns.us-west-2.amazonaws.com/?Action=ConfirmSubscription&TopicArn=arn:aws:sns:us-west-2:123456789012:MyTopic&Token=2336412f37fb687f5d51e6e241d09c805a5a57b30d712f794cc5f6a988666d92768dd60a747ba6f3beb71854e285d6ad02428b09ceece29417f1f02d609c582afbacc99c583a916b9981dd2728f4ae6fdb82efd087cc3b7849e05798d2d2785c03b0879594eeac82c01f235d0e717736",
  "Timestamp" : "2012-04-26T20:45:04.751Z",
  "SignatureVersion" : "1",
  "Signature" : "EXAMPLEpH+DcEwjAPg8O9mY8dReBSwksfg2S7WKQcikcNKWLQjwu6A4VbeS0QHVCkhRS7fUQvi2egU3N858fiTDN6bkkOxYDVrY0Ad8L10Hs3zH81mtnPk5uvvolIC1CXGu43obcgFxeL3khZl8IKvO61GWB6jI9b5+gLPoBc1Q=",
  "SigningCertURL" : "https://sns.us-west-2.amazonaws.com/SimpleNotificationService-f3ecfb7224c7233fe7bb5f59f96de52f.pem"
}
```

Unfortunately, this message currently does not happen. Thus breaking the AWS SNS contract.


http://docs.aws.amazon.com/sns/latest/dg/SendMessageToHttp.html 
https://docs.aws.amazon.com/sns/latest/dg/json-formats.html#http-subscription-confirmation-json


Discussion:
👍 
I appear to be running into the same issue. Is this supposed to be fixed? My endpoint is not called when subscribing but it is called when publishing.

Resolving patch:
diff --git a/localstack/services/generic_proxy.py b/localstack/services/generic_proxy.py
--- a/localstack/services/generic_proxy.py
+++ b/localstack/services/generic_proxy.py
@@ -67,7 +67,7 @@ def forward_request(self, method, path, data, headers):
         """
         return True
 
-    def return_response(self, method, path, data, headers, response):
+    def return_response(self, method, path, data, headers, response, request_handler=None):
         """ This interceptor method is called by the proxy when returning a response
             (*after* having forwarded the request and received a response from the backend
             service). It receives details of the incoming request as well as the response
diff --git a/localstack/services/sns/sns_listener.py b/localstack/services/sns/sns_listener.py
--- a/localstack/services/sns/sns_listener.py
+++ b/localstack/services/sns/sns_listener.py
@@ -1,5 +1,4 @@
 import ast
-import datetime
 import json
 import uuid
 import logging
@@ -8,9 +7,10 @@
 import xmltodict
 from requests.models import Response, Request
 from six.moves.urllib import parse as urlparse
+from localstack.config import TEST_SNS_URL
 from localstack.constants import TEST_AWS_ACCOUNT_ID, MOTO_ACCOUNT_ID
 from localstack.utils.aws import aws_stack
-from localstack.utils.common import short_uid, to_str
+from localstack.utils.common import short_uid, to_str, timestamp, TIMESTAMP_FORMAT_MILLIS
 from localstack.utils.analytics import event_publisher
 from localstack.services.awslambda import lambda_api
 from localstack.services.generic_proxy import ProxyListener
@@ -166,10 +166,12 @@ def return_response(self, method, path, data, headers, response):
 UPDATE_SNS = ProxyListenerSNS()
 
 
-def publish_message(topic_arn, req_data):
+def publish_message(topic_arn, req_data, subscription_arn=None):
     message = req_data['Message'][0]
     sqs_client = aws_stack.connect_to_service('sqs')
     for subscriber in SNS_SUBSCRIPTIONS.get(topic_arn, []):
+        if subscription_arn not in [None, subscriber['SubscriptionArn']]:
+            continue
         filter_policy = json.loads(subscriber.get('FilterPolicy', '{}'))
         message_attributes = get_message_attributes(req_data)
         if not check_filter_policy(filter_policy, message_attributes):
@@ -212,8 +214,8 @@ def publish_message(topic_arn, req_data):
                     'Content-Type': 'text/plain',
                     'x-amz-sns-message-type': 'Notification'
                 },
-                data=message_body
-            )
+                data=message_body,
+                verify=False)
         else:
             LOGGER.warning('Unexpected protocol "%s" for SNS subscription' % subscriber['Protocol'])
 
@@ -244,6 +246,18 @@ def do_subscribe(topic_arn, endpoint, protocol, subscription_arn, attributes):
     subscription.update(attributes)
     SNS_SUBSCRIPTIONS[topic_arn].append(subscription)
 
+    # Send out confirmation message for HTTP(S), fix for https://github.com/localstack/localstack/issues/881
+    if protocol in ['http', 'https']:
+        token = short_uid()
+        confirmation = {
+            'Type': ['SubscriptionConfirmation'],
+            'Token': [token],
+            'Message': [('You have chosen to subscribe to the topic %s.\n' % topic_arn) +
+                'To confirm the subscription, visit the SubscribeURL included in this message.'],
+            'SubscribeURL': ['%s/?Action=ConfirmSubscription&TopicArn=%s&Token=%s' % (TEST_SNS_URL, topic_arn, token)]
+        }
+        publish_message(topic_arn, confirmation, subscription_arn)
+
 
 def do_unsubscribe(subscription_arn):
     for topic_arn in SNS_SUBSCRIPTIONS:
@@ -335,8 +349,8 @@ def create_sns_message_body(subscriber, req_data):
 
     data = {}
     data['MessageId'] = str(uuid.uuid4())
-    data['Type'] = 'Notification'
-    data['Timestamp'] = datetime.datetime.utcnow().isoformat(timespec='milliseconds') + 'Z'
+    data['Type'] = req_data.get('Type', ['Notification'])[0]
+    data['Timestamp'] = timestamp(format=TIMESTAMP_FORMAT_MILLIS)
     data['Message'] = message
     data['TopicArn'] = subscriber['TopicArn']
     if subject is not None:
@@ -344,6 +358,10 @@ def create_sns_message_body(subscriber, req_data):
     attributes = get_message_attributes(req_data)
     if attributes:
         data['MessageAttributes'] = attributes
+    for attr in ['Token', 'SubscribeURL']:
+        value = req_data.get(attr, [None])[0]
+        if value:
+            data[attr] = value
     result = json.dumps(data)
     return result
 


Test patch:
diff --git a/tests/integration/test_lambda.py b/tests/integration/test_lambda.py
--- a/tests/integration/test_lambda.py
+++ b/tests/integration/test_lambda.py
@@ -4,7 +4,6 @@
 import time
 import unittest
 from io import BytesIO
-from requests.models import Response
 from localstack import config
 from localstack.constants import LOCALSTACK_ROOT_FOLDER, LOCALSTACK_MAVEN_VERSION
 from localstack.utils import testutil
@@ -85,10 +84,7 @@ def test_forward_to_fallback_url_http(self):
         class MyUpdateListener(ProxyListener):
             def forward_request(self, method, path, data, headers):
                 records.append(data)
-                response = Response()
-                response.status_code = 200
-                response._content = ''
-                return response
+                return 200
 
         records = []
         local_port = get_free_tcp_port()
diff --git a/tests/integration/test_sns.py b/tests/integration/test_sns.py
--- a/tests/integration/test_sns.py
+++ b/tests/integration/test_sns.py
@@ -2,11 +2,11 @@
 
 import json
 import unittest
-
 from botocore.exceptions import ClientError
-
 from localstack.utils.aws import aws_stack
-from localstack.utils.common import to_str
+from localstack.utils.common import to_str, get_free_tcp_port, retry, wait_for_port_open, get_service_protocol
+from localstack.services.infra import start_proxy
+from localstack.services.generic_proxy import ProxyListener
 
 TEST_TOPIC_NAME = 'TestTopic_snsTest'
 TEST_QUEUE_NAME = 'TestQueue_snsTest'
@@ -38,6 +38,26 @@ def test_publish_unicode_chars(self):
         msg_received = msg_received['Message']
         self.assertEqual(message, msg_received)
 
+    def test_subscribe_http_endpoint(self):
+        # create HTTP endpoint and connect it to SNS topic
+        class MyUpdateListener(ProxyListener):
+            def forward_request(self, method, path, data, headers):
+                records.append(json.loads(to_str(data)))
+                return 200
+
+        records = []
+        local_port = get_free_tcp_port()
+        proxy = start_proxy(local_port, backend_url=None, update_listener=MyUpdateListener())
+        wait_for_port_open(local_port)
+        queue_arn = '%s://localhost:%s' % (get_service_protocol(), local_port)
+        self.sns_client.subscribe(TopicArn=self.topic_arn, Protocol='http', Endpoint=queue_arn)
+
+        def received():
+            assert records[0]['Type'] == 'SubscriptionConfirmation'
+
+        retry(received, retries=5, sleep=1)
+        proxy.stop()
+
     def test_attribute_raw_subscribe(self):
         # create SNS topic and connect it to an SQS queue
         queue_arn = aws_stack.sqs_queue_arn(TEST_QUEUE_NAME)
@@ -108,9 +128,7 @@ def test_tags(self):
 
         self.sns_client.untag_resource(
             ResourceArn=self.topic_arn,
-            TagKeys=[
-                '123',
-            ]
+            TagKeys=['123']
         )
 
         tags = self.sns_client.list_tags_for_resource(ResourceArn=self.topic_arn)
diff --git a/tests/unit/test_sns_listener.py b/tests/unit/test_sns_listener.py
--- a/tests/unit/test_sns_listener.py
+++ b/tests/unit/test_sns_listener.py
@@ -33,8 +33,8 @@ def test_unsubscribe_should_remove_listener(self):
         self.assertTrue(sns_listener.get_topic_by_arn(topic_arn) is not None)
         sns_listener.do_subscribe(
             topic_arn,
-            'http://localhost:1234/listen',
-            'http',
+            'arn:aws:sqs:us-east-1:123456789012:test-queue',
+            'sqs',
             sub_arn,
             {}
         )
@@ -181,23 +181,15 @@ def test_only_one_subscription_per_topic_per_endpoint(self):
         self.assertFalse(sns_listener.get_topic_by_arn(topic_arn))
         sns_listener.do_create_topic(topic_arn)
         self.assertTrue(sns_listener.get_topic_by_arn(topic_arn) is not None)
-        sns_listener.do_subscribe(
-            topic_arn,
-            'http://localhost:1234/listen',
-            'http',
-            sub_arn,
-            {}
-        )
-
-        self.assertEqual(len(sns_listener.SNS_SUBSCRIPTIONS[topic_arn]), 1)
-        sns_listener.do_subscribe(
-            topic_arn,
-            'http://localhost:1234/listen',
-            'http',
-            sub_arn,
-            {}
-        )
-        self.assertEqual(len(sns_listener.SNS_SUBSCRIPTIONS[topic_arn]), 1)
+        for i in [1, 2]:
+            sns_listener.do_subscribe(
+                topic_arn,
+                'arn:aws:sqs:us-east-1:123456789012:test-queue-1',
+                'sqs',
+                sub_arn,
+                {}
+            )
+            self.assertEqual(len(sns_listener.SNS_SUBSCRIPTIONS[topic_arn]), 1)
 
 
 def test_filter_policy():