Repository: localstack/localstack

Issue:
Missing Sns Subject at LambdaExecutor
Hello,

I have an issue invoking a java Lambda subscribed to and SNS topic, when i publish an SNS message with a subject, the subject missing.

I have narrowed the issue down to LambdaExecutor

TIA!



Discussion:

Resolving patch:
diff --git a/localstack/services/awslambda/lambda_api.py b/localstack/services/awslambda/lambda_api.py
--- a/localstack/services/awslambda/lambda_api.py
+++ b/localstack/services/awslambda/lambda_api.py
@@ -123,7 +123,6 @@ def get_response(self):
 
 
 class LambdaContext(object):
-
     def __init__(self, func_details, qualifier=None):
         self.function_name = func_details.name()
         self.function_version = func_details.get_qualifier_version(qualifier)
@@ -226,13 +225,13 @@ def process_apigateway_invocation(func_arn, path, payload, headers={},
         LOG.warning('Unable to run Lambda function on API Gateway message: %s %s' % (e, traceback.format_exc()))
 
 
-def process_sns_notification(func_arn, topic_arn, subscriptionArn, message, message_attributes, subject='',):
+def process_sns_notification(func_arn, topic_arn, subscription_arn, message, message_attributes, subject='',):
     try:
         event = {
             'Records': [{
                 'EventSource': 'localstack:sns',
                 'EventVersion': '1.0',
-                'EventSubscriptionArn': subscriptionArn,
+                'EventSubscriptionArn': subscription_arn,
                 'Sns': {
                     'Type': 'Notification',
                     'TopicArn': topic_arn,
@@ -513,7 +512,6 @@ def execute(event, context):
 
 
 def set_archive_code(code, lambda_name, zip_file_content=None):
-
     # get metadata
     lambda_arn = func_arn(lambda_name)
     lambda_details = arn_to_lambda[lambda_arn]
@@ -550,7 +548,6 @@ def set_archive_code(code, lambda_name, zip_file_content=None):
 
 
 def set_function_code(code, lambda_name, lambda_cwd=None):
-
     def generic_handler(event, context):
         raise ClientError(('Unable to find executor for Lambda function "%s". Note that ' +
             'Node.js, Golang, and .Net Core Lambdas currently require LAMBDA_EXECUTOR=docker') % lambda_name)
@@ -600,7 +597,6 @@ def generic_handler(event, context):
 
     # Obtain handler details for any non-Java Lambda function
     if not is_java:
-
         handler_file = get_handler_file_from_name(handler_name, runtime=runtime)
         handler_function = get_handler_function_from_name(handler_name, runtime=runtime)
 
@@ -633,7 +629,6 @@ def generic_handler(event, context):
                 raise ClientError('Unable to get handler function from lambda code.', e)
 
     add_function_mapping(lambda_name, lambda_handler, lambda_cwd)
-
     return {'FunctionName': lambda_name}
 
 
@@ -677,7 +672,7 @@ def format_func_details(func_details, version=None, always_add_version=False):
             'Variables': func_details.envvars
         }
     if (always_add_version or version != '$LATEST') and len(result['FunctionArn'].split(':')) <= 7:
-        result['FunctionArn'] += ':%s' % (version)
+        result['FunctionArn'] += ':%s' % version
     return result
 
 
diff --git a/localstack/services/awslambda/lambda_executors.py b/localstack/services/awslambda/lambda_executors.py
--- a/localstack/services/awslambda/lambda_executors.py
+++ b/localstack/services/awslambda/lambda_executors.py
@@ -78,15 +78,23 @@ def is_nodejs_runtime(lambda_details):
     return runtime.startswith('nodejs')
 
 
+def _store_logs(func_details, log_output, invocation_time=None, container_id=None):
+    log_group_name = '/aws/lambda/%s' % func_details.name()
+    container_id = container_id or short_uid()
+    invocation_time = invocation_time or int(time.time() * 1000)
+    invocation_time_secs = int(invocation_time / 1000)
+    time_str = time.strftime('%Y/%m/%d', time.gmtime(invocation_time_secs))
+    log_stream_name = '%s/[LATEST]%s' % (time_str, container_id)
+    return store_cloudwatch_logs(log_group_name, log_stream_name, log_output, invocation_time)
+
+
 class LambdaExecutor(object):
     """ Base class for Lambda executors. Subclasses must overwrite the _execute method """
-
     def __init__(self):
         # keeps track of each function arn and the last time it was invoked
         self.function_invoke_times = {}
 
     def execute(self, func_arn, func_details, event, context=None, version=None, asynchronous=False):
-
         def do_execute(*args):
             # set the invocation time in milliseconds
             invocation_time = int(time.time() * 1000)
@@ -111,8 +119,7 @@ def do_execute(*args):
 
         # Inform users about asynchronous mode of the lambda execution.
         if asynchronous:
-            LOG.debug('Lambda executed in Event (asynchronous) mode, no response from this '
-                      'function will be returned to caller')
+            LOG.debug('Lambda executed in Event (asynchronous) mode, no response will be returned to caller')
             FuncThread(do_execute).start()
             return None, 'Lambda executed asynchronously.'
 
@@ -128,15 +135,6 @@ def startup(self):
     def cleanup(self, arn=None):
         pass
 
-    def _store_logs(self, func_details, log_output, invocation_time=None, container_id=None):
-        log_group_name = '/aws/lambda/%s' % func_details.name()
-        container_id = container_id or short_uid()
-        invocation_time = invocation_time or int(time.time() * 1000)
-        invocation_time_secs = int(invocation_time / 1000)
-        time_str = time.strftime('%Y/%m/%d', time.gmtime(invocation_time_secs))
-        log_stream_name = '%s/[$LATEST]%s' % (time_str, container_id)
-        return store_cloudwatch_logs(log_group_name, log_stream_name, log_output, invocation_time)
-
     def run_lambda_executor(self, cmd, event=None, func_details=None, env_vars={}):
         process = run(cmd, asynchronous=True, stderr=subprocess.PIPE, outfile=subprocess.PIPE,
                       env_vars=env_vars, stdin=True)
@@ -159,7 +157,7 @@ def run_lambda_executor(self, cmd, event=None, func_details=None, env_vars={}):
         LOG.debug('Lambda %s result / log output:\n%s\n> %s' % (func_arn, result.strip(), log_formatted))
 
         # store log output - TODO get live logs from `process` above?
-        self._store_logs(func_details, log_output)
+        _store_logs(func_details, log_output)
 
         if return_code != 0:
             raise Exception('Lambda process returned error status code: %s. Result: %s. Output:\n%s' %
@@ -194,7 +192,6 @@ def prepare_event(self, environment, event_body):
         return None
 
     def _execute(self, func_arn, func_details, event, context=None, version=None):
-
         lambda_cwd = func_details.cwd
         runtime = func_details.runtime
         handler = func_details.handler
@@ -257,7 +254,6 @@ def _execute(self, func_arn, func_details, event, context=None, version=None):
 
 class LambdaExecutorReuseContainers(LambdaExecutorContainers):
     """ Executor class for executing Lambda functions in re-usable Docker containers """
-
     def __init__(self):
         super(LambdaExecutorReuseContainers, self).__init__()
         # locking thread for creation/destruction of docker containers.
@@ -272,7 +268,6 @@ def __init__(self):
         self.port_offset = LAMBDA_SERVER_PORT_OFFSET
 
     def prepare_execution(self, func_arn, env_vars, runtime, command, handler, lambda_cwd):
-
         # check whether the Lambda has been invoked before
         has_been_invoked_before = func_arn in self.function_invoke_times
 
@@ -520,11 +515,8 @@ def get_docker_container_network(self, func_arn):
         :param func_arn: The ARN of the lambda function.
         :return: name of the container network
         """
-
         with self.docker_container_lock:
-
             status = self.get_docker_container_status(func_arn)
-
             # container does not exist
             if status == 0:
                 return ''
@@ -584,7 +576,6 @@ def get_container_name(self, func_arn):
 
 
 class LambdaExecutorSeparateContainers(LambdaExecutorContainers):
-
     def __init__(self):
         super(LambdaExecutorSeparateContainers, self).__init__()
         self.next_port = 1
@@ -592,7 +583,6 @@ def __init__(self):
         self.port_offset = LAMBDA_API_PORT_OFFSET
 
     def prepare_event(self, environment, event_body):
-
         # Tell Lambci to use STDIN for the event
         environment['DOCKER_LAMBDA_USE_STDIN'] = '1'
         return event_body.encode()
@@ -650,7 +640,6 @@ def prepare_execution(self, func_arn, env_vars, runtime, command, handler, lambd
 
 
 class LambdaExecutorLocal(LambdaExecutor):
-
     def _execute(self, func_arn, func_details, event, context=None, version=None):
         lambda_cwd = func_details.cwd
         environment = func_details.envvars.copy()
@@ -680,7 +669,7 @@ def do_execute():
                 log_output += ('\n' if log_output else '') + stream
 
         # store logs to CloudWatch
-        self._store_logs(func_details, log_output)
+        _store_logs(func_details, log_output)
 
         return result
 
diff --git a/localstack/utils/cloudformation/template_deployer.py b/localstack/utils/cloudformation/template_deployer.py
--- a/localstack/utils/cloudformation/template_deployer.py
+++ b/localstack/utils/cloudformation/template_deployer.py
@@ -441,11 +441,11 @@ def replace(params, **kwargs):
     }
 }
 
-
 # ----------------
 # UTILITY METHODS
 # ----------------
 
+
 def convert_acl_cf_to_s3(acl):
     """ Convert a CloudFormation ACL string (e.g., 'PublicRead') to an S3 ACL string (e.g., 'public-read') """
     return re.sub('(?<!^)(?=[A-Z])', '-', acl).lower()


Test patch:
diff --git a/tests/integration/lambdas/lambda_echo.py b/tests/integration/lambdas/lambda_echo.py
new file mode 100644
--- /dev/null
+++ b/tests/integration/lambdas/lambda_echo.py
@@ -0,0 +1,7 @@
+import json
+
+
+def handler(event, context):
+    # Just print the event was passed to lambda
+    print(json.dumps(event))
+    return 0
diff --git a/tests/integration/test_lambda.py b/tests/integration/test_lambda.py
--- a/tests/integration/test_lambda.py
+++ b/tests/integration/test_lambda.py
@@ -34,6 +34,7 @@
 TEST_LAMBDA_JAVA_WITH_LIB = os.path.join(THIS_FOLDER, 'lambdas', 'java', 'lambda-function-with-lib-0.0.1.jar')
 TEST_LAMBDA_ENV = os.path.join(THIS_FOLDER, 'lambdas', 'lambda_environment.py')
 
+
 TEST_LAMBDA_NAME_PY = 'test_lambda_py'
 TEST_LAMBDA_NAME_PY3 = 'test_lambda_py3'
 TEST_LAMBDA_NAME_JS = 'test_lambda_js'
@@ -45,16 +46,32 @@
 TEST_LAMBDA_NAME_JAVA_SERIALIZABLE = 'test_lambda_java_serializable'
 TEST_LAMBDA_NAME_ENV = 'test_lambda_env'
 
+TEST_LAMBDA_ECHO_FILE = os.path.join(THIS_FOLDER, 'lambdas', 'lambda_echo.py')
+TEST_LAMBDA_FUNCTION_PREFIX = 'lambda-function'
+TEST_SNS_TOPIC_NAME = 'sns-topic-1'
+
 MAVEN_BASE_URL = 'https://repo.maven.apache.org/maven2'
-TEST_LAMBDA_JAR_URL = ('{url}/cloud/localstack/{name}/{version}/{name}-{version}-tests.jar').format(
+
+TEST_LAMBDA_JAR_URL = '{url}/cloud/localstack/{name}/{version}/{name}-{version}-tests.jar'.format(
     version=LOCALSTACK_MAVEN_VERSION, url=MAVEN_BASE_URL, name='localstack-utils')
 
-TEST_LAMBDA_LIBS = ['localstack', 'localstack_client', 'requests',
-    'psutil', 'urllib3', 'chardet', 'certifi', 'idna', 'pip', 'dns']
+TEST_LAMBDA_LIBS = [
+    'localstack', 'localstack_client', 'requests', 'psutil', 'urllib3', 'chardet', 'certifi', 'idna', 'pip', 'dns'
+]
 
 
-class LambdaTestBase(unittest.TestCase):
+def _run_forward_to_fallback_url(url, num_requests=3):
+    lambda_client = aws_stack.connect_to_service('lambda')
+    config.LAMBDA_FALLBACK_URL = url
+    try:
+        for i in range(num_requests):
+            lambda_client.invoke(FunctionName='non-existing-lambda-%s' % i,
+                Payload=b'{}', InvocationType='RequestResponse')
+    finally:
+        config.LAMBDA_FALLBACK_URL = ''
+
 
+class LambdaTestBase(unittest.TestCase):
     def check_lambda_logs(self, func_name, expected_lines=[]):
         logs_client = aws_stack.connect_to_service('logs')
         log_group_name = '/aws/lambda/%s' % func_name
@@ -72,7 +89,6 @@ def check_lambda_logs(self, func_name, expected_lines=[]):
 
 
 class TestLambdaBaseFeatures(unittest.TestCase):
-
     def test_forward_to_fallback_url_dynamodb(self):
         db_table = 'lambda-records'
         ddb_client = aws_stack.connect_to_service('dynamodb')
@@ -81,7 +97,7 @@ def num_items():
             return len((run_safe(ddb_client.scan, TableName=db_table) or {'Items': []})['Items'])
 
         items_before = num_items()
-        self._run_forward_to_fallback_url('dynamodb://%s' % db_table)
+        _run_forward_to_fallback_url('dynamodb://%s' % db_table)
         items_after = num_items()
         self.assertEqual(items_after, items_before + 3)
 
@@ -96,21 +112,11 @@ def forward_request(self, method, path, data, headers):
         proxy = start_proxy(local_port, backend_url=None, update_listener=MyUpdateListener())
 
         items_before = len(records)
-        self._run_forward_to_fallback_url('%s://localhost:%s' % (get_service_protocol(), local_port))
+        _run_forward_to_fallback_url('%s://localhost:%s' % (get_service_protocol(), local_port))
         items_after = len(records)
         self.assertEqual(items_after, items_before + 3)
         proxy.stop()
 
-    def _run_forward_to_fallback_url(self, url, num_requests=3):
-        lambda_client = aws_stack.connect_to_service('lambda')
-        config.LAMBDA_FALLBACK_URL = url
-        try:
-            for i in range(num_requests):
-                lambda_client.invoke(FunctionName='non-existing-lambda-%s' % i,
-                    Payload=b'{}', InvocationType='RequestResponse')
-        finally:
-            config.LAMBDA_FALLBACK_URL = ''
-
     def test_dead_letter_queue(self):
         sqs_client = aws_stack.connect_to_service('sqs')
         lambda_client = aws_stack.connect_to_service('lambda')
@@ -176,6 +182,8 @@ class TestPythonRuntimes(LambdaTestBase):
     def setUpClass(cls):
         cls.lambda_client = aws_stack.connect_to_service('lambda')
         cls.s3_client = aws_stack.connect_to_service('s3')
+        cls.sns_client = aws_stack.connect_to_service('sns')
+
         Util.create_function(TEST_LAMBDA_PYTHON, TEST_LAMBDA_NAME_PY,
             runtime=LAMBDA_RUNTIME_PYTHON27, libs=TEST_LAMBDA_LIBS)
 
@@ -374,6 +382,74 @@ def test_handler_in_submodule(self):
         self.assertEqual(result['StatusCode'], 200)
         self.assertEqual(result_data['event'], json.loads('{}'))
 
+    def test_lambda_subscribe_sns_topic(self):
+        function_name = '{}-{}'.format(TEST_LAMBDA_FUNCTION_PREFIX, short_uid())
+
+        zip_file = testutil.create_lambda_archive(
+            script=load_file(TEST_LAMBDA_ECHO_FILE),
+            get_content=True,
+            runtime=LAMBDA_RUNTIME_PYTHON36
+        )
+
+        testutil.create_lambda_function(
+            zip_file=zip_file,
+            func_name=function_name,
+            runtime=LAMBDA_RUNTIME_PYTHON36
+        )
+
+        topic = self.sns_client.create_topic(
+            Name=TEST_SNS_TOPIC_NAME
+        )
+        topic_arn = topic['TopicArn']
+
+        self.sns_client.subscribe(
+            TopicArn=topic_arn,
+            Protocol='lambda',
+            Endpoint=lambda_api.func_arn(function_name),
+        )
+
+        subject = '[Subject] Test subject'
+        message = 'Hello world.'
+        self.sns_client.publish(
+            TopicArn=topic_arn,
+            Subject=subject,
+            Message=message
+        )
+
+        logs = aws_stack.connect_to_service('logs')
+
+        def get_event_message(events):
+            for event in events:
+                raw_message = event['message']
+                if 'START' in raw_message or 'END' in raw_message or 'REPORT' in raw_message:
+                    continue
+
+                return json.loads(raw_message)
+
+            return None
+
+        # wait for lambda executing
+        def check_log_streams():
+            rs = logs.describe_log_streams(
+                logGroupName='/aws/lambda/{}'.format(function_name)
+            )
+
+            self.assertEqual(len(rs['logStreams']), 1)
+            return rs['logStreams'][0]['logStreamName']
+
+        log_stream = retry(check_log_streams, retries=3, sleep=2)
+        rs = logs.get_log_events(
+            logGroupName='/aws/lambda/{}'.format(function_name),
+            logStreamName=log_stream
+        )
+
+        message = get_event_message(rs['events'])
+        self.assertEqual(len(message['Records']), 1)
+        notification = message['Records'][0]['Sns']
+
+        self.assertIn('Subject', notification)
+        self.assertEqual(notification['Subject'], subject)
+
 
 class TestNodeJSRuntimes(LambdaTestBase):
     @classmethod
diff --git a/tests/unit/test_lambda.py b/tests/unit/test_lambda.py
--- a/tests/unit/test_lambda.py
+++ b/tests/unit/test_lambda.py
@@ -604,12 +604,11 @@ def test_get_java_lib_folder_classpath_archive_is_None(self):
     @mock.patch('localstack.services.awslambda.lambda_executors.store_cloudwatch_logs')
     def test_executor_store_logs_can_handle_milliseconds(self, mock_store_cloudwatch_logs):
         mock_details = mock.Mock()
-        executor = lambda_executors.LambdaExecutor()
         t_sec = time.time()  # plain old epoch secs
         t_ms = time.time() * 1000  # epoch ms as a long-int like AWS
 
         # pass t_ms millisecs to _store_logs
-        executor._store_logs(mock_details, 'mock log output', t_ms)
+        lambda_executors._store_logs(mock_details, 'mock log output', t_ms)
 
         # expect the computed log-stream-name to having a prefix matching the date derived from t_sec
         today = datetime.datetime.utcfromtimestamp(t_sec).strftime('%Y/%m/%d')