Repository: localstack/localstack

Issue:
Lambdas don't trigger from SQS messages if SendMessageBatch is used
It appears lambdas are not triggered from items being pushed to SQS if [SendMessageBatch](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessageBatch.html) is used instead of [SendMessage](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html)


Discussion:
@taylorfinnell, not answering your question, but I want to know. I would like to subscribe my local lambda service to a local aws service(dynamo db change feed). I searched for a way to so but couldn't find one. Can I ask how you could make a subscrition for your lambda service? Thank you :)
I've faced the same issue. I was publishing a batch of messages to the queue that was supposed to trigger the lambda. It just worked when I changed to publish a single message.
Encountered the same problem. Was trying to determine if this is standard AWS behavior or specific to localstack. Going to assume specific to localstack although have not tried with actual AWS queues and lambdas yet to be 100% sure...
Thanks for reporting @taylorfinnell @cosmir17 @wapmesquita @scott-mcdonald . Would any of you be able to pick this up and create a small pull requests with a simple test? That would be highly appreciated - thanks!

The `_send_message(..)` method in `sqs_listener.py` would be a good starting point: https://github.com/localstack/localstack/blob/master/localstack/services/sqs/sqs_listener.py#L261-L280 It could be extended to also accept a batch of messages.
A co-worker implemented a fix for this on a private fork this past summer, but we never got around to opening up a PR. I just took a glance and it seems like the `sqs_listener.py` and `lambda_api.py` logic has changed a fair amount from when the fork happened. I'm working on rebasing on master so I'll attempt to get a PR up for this.

Resolving patch:
diff --git a/localstack/services/awslambda/lambda_api.py b/localstack/services/awslambda/lambda_api.py
--- a/localstack/services/awslambda/lambda_api.py
+++ b/localstack/services/awslambda/lambda_api.py
@@ -330,13 +330,15 @@ def listener_loop(*args):
                 for source in sources:
                     queue_arn = source['EventSourceArn']
                     lambda_arn = source['FunctionArn']
+                    batch_size = max(min(source.get('BatchSize', 1), 10), 1)
 
                     try:
                         region_name = queue_arn.split(':')[3]
                         queue_url = aws_stack.sqs_queue_url_for_arn(queue_arn)
                         result = sqs_client.receive_message(
                             QueueUrl=queue_url,
-                            MessageAttributeNames=['All']
+                            MessageAttributeNames=['All'],
+                            MaxNumberOfMessages=batch_size
                         )
                         messages = result.get('Messages')
                         if not messages:
diff --git a/localstack/services/sqs/sqs_listener.py b/localstack/services/sqs/sqs_listener.py
--- a/localstack/services/sqs/sqs_listener.py
+++ b/localstack/services/sqs/sqs_listener.py
@@ -269,7 +269,7 @@ def return_response(self, method, path, data, headers, response, request_handler
                 _set_queue_attributes(queue_url, req_data)
 
         # instruct listeners to fetch new SQS message
-        if action == 'SendMessage':
+        if action in ('SendMessage', 'SendMessageBatch'):
             _process_sent_message(path, req_data, headers)
 
         if content_str_original != content_str:
diff --git a/localstack/utils/testutil.py b/localstack/utils/testutil.py
--- a/localstack/utils/testutil.py
+++ b/localstack/utils/testutil.py
@@ -162,15 +162,23 @@ def create_lambda_function(func_name, zip_file=None, event_source_arn=None, hand
     kwargs.update(additional_kwargs)
     if layers:
         kwargs['Layers'] = layers
-    client.create_function(**kwargs)
+    create_func_resp = client.create_function(**kwargs)
+
+    resp = {
+        'CreateFunctionResponse': create_func_resp,
+        'CreateEventSourceMappingResponse': None
+    }
+
     # create event source mapping
     if event_source_arn:
-        client.create_event_source_mapping(
+        resp['CreateEventSourceMappingResponse'] = client.create_event_source_mapping(
             FunctionName=func_name,
             EventSourceArn=event_source_arn,
             StartingPosition=starting_position
         )
 
+    return resp
+
 
 def assert_objects(asserts, all_objects):
     if type(asserts) is not list:


Test patch:
diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py
--- a/tests/integration/test_integration.py
+++ b/tests/integration/test_integration.py
@@ -14,7 +14,7 @@
 from localstack.utils.kinesis import kinesis_connector
 from localstack.utils.aws import aws_stack
 from .lambdas import lambda_integration
-from .test_lambda import TEST_LAMBDA_PYTHON, TEST_LAMBDA_LIBS
+from .test_lambda import TEST_LAMBDA_PYTHON, TEST_LAMBDA_PYTHON_ECHO, TEST_LAMBDA_LIBS
 
 TEST_STREAM_NAME = lambda_integration.KINESIS_STREAM_NAME
 TEST_LAMBDA_SOURCE_STREAM_NAME = 'test_source_stream'
@@ -22,6 +22,7 @@
 TEST_LAMBDA_NAME_DDB = 'test_lambda_ddb'
 TEST_LAMBDA_NAME_STREAM = 'test_lambda_stream'
 TEST_LAMBDA_NAME_QUEUE = 'test_lambda_queue'
+TEST_LAMBDA_NAME_QUEUE_BATCH = 'test_lambda_queue_batch'
 TEST_FIREHOSE_NAME = 'test_firehose'
 TEST_BUCKET_NAME = lambda_integration.TEST_BUCKET_NAME
 TEST_TOPIC_NAME = 'test_topic'
@@ -291,6 +292,8 @@ def check_events():
         # clean up
         testutil.delete_lambda_function(TEST_LAMBDA_NAME_STREAM)
         testutil.delete_lambda_function(TEST_LAMBDA_NAME_DDB)
+        testutil.delete_lambda_function(TEST_LAMBDA_NAME_QUEUE)
+        sqs.delete_queue(QueueUrl=sqs_queue_info['QueueUrl'])
 
     def test_lambda_streams_batch_and_transactions(self):
         ddb_lease_table_suffix = '-kclapp2'
@@ -470,6 +473,97 @@ def test_kinesis_lambda_forward_chain(self):
         kinesis.delete_stream(StreamName=TEST_CHAIN_STREAM1_NAME)
         kinesis.delete_stream(StreamName=TEST_CHAIN_STREAM2_NAME)
 
+    def test_sqs_batch_lambda_forward(self):
+        sqs = aws_stack.connect_to_service('sqs')
+        lambda_api = aws_stack.connect_to_service('lambda')
+
+        # deploy test lambda connected to SQS queue
+        sqs_queue_info = testutil.create_sqs_queue(TEST_LAMBDA_NAME_QUEUE_BATCH)
+        queue_url = sqs_queue_info['QueueUrl']
+        zip_file = testutil.create_lambda_archive(
+            load_file(TEST_LAMBDA_PYTHON_ECHO),
+            get_content=True,
+            libs=TEST_LAMBDA_LIBS,
+            runtime=LAMBDA_RUNTIME_PYTHON27
+        )
+        resp = testutil.create_lambda_function(
+            func_name=TEST_LAMBDA_NAME_QUEUE_BATCH,
+            zip_file=zip_file,
+            event_source_arn=sqs_queue_info['QueueArn'],
+            runtime=LAMBDA_RUNTIME_PYTHON27
+        )
+
+        event_source_id = resp['CreateEventSourceMappingResponse']['UUID']
+        lambda_api.update_event_source_mapping(
+            UUID=event_source_id,
+            BatchSize=5
+        )
+
+        messages_to_send = [
+            {
+                'Id': 'message{:02d}'.format(i),
+                'MessageBody': 'msgBody{:02d}'.format(i),
+                'MessageAttributes': {
+                    'CustomAttribute': {
+                        'DataType': 'String',
+                        'StringValue': 'CustomAttributeValue{:02d}'.format(i)
+                    }
+                }
+            }
+            for i in range(1, 12)
+        ]
+
+        start_time = datetime.now()
+
+        # send 11 messages (which should get split into 3 batches)
+        sqs.send_message_batch(QueueUrl=queue_url, Entries=messages_to_send[:10])
+        sqs.send_message(
+            QueueUrl=queue_url,
+            MessageBody=messages_to_send[10]['MessageBody'],
+            MessageAttributes=messages_to_send[10]['MessageAttributes']
+        )
+
+        def wait_for_done():
+            attributes = sqs.get_queue_attributes(
+                QueueUrl=queue_url,
+                AttributeNames=[
+                    'ApproximateNumberOfMessages',
+                    'ApproximateNumberOfMessagesDelayed',
+                    'ApproximateNumberOfMessagesNotVisible'
+                ],
+            )['Attributes']
+            msg_count = int(attributes.get('ApproximateNumberOfMessages'))
+            self.assertEqual(msg_count, 0, 'expecting queue to be empty')
+
+            delayed_count = int(attributes.get('ApproximateNumberOfMessagesDelayed'))
+            if delayed_count != 0:
+                LOGGER.warning(('SQS delayed message count (actual/expected): %s/%s') %
+                    (delayed_count, 0))
+
+            not_visible_count = int(attributes.get('ApproximateNumberOfMessagesNotVisible'))
+            if not_visible_count != 0:
+                LOGGER.warning(('SQS messages not visible (actual/expected): %s/%s') %
+                    (not_visible_count, 0))
+
+            invocation_count = get_lambda_invocations_count(
+                TEST_LAMBDA_NAME_QUEUE_BATCH,
+                period=120,
+                start_time=start_time,
+                end_time=datetime.now()
+            )
+            if invocation_count != 3:
+                LOGGER.warning(('Lambda invocations (actual/expected): %s/%s') %
+                    (invocation_count, 3))
+
+            self.assertEqual(delayed_count, 0, 'no messages waiting for retry')
+            self.assertEqual(delayed_count + not_visible_count, 0, 'no in flight messages')
+            self.assertEqual(invocation_count, 3, 'expected 3 lambda invocations')
+
+        # wait for the queue to drain (max 90s)
+        retry(wait_for_done, retries=18, sleep=5.0)
+
+        testutil.delete_lambda_function(TEST_LAMBDA_NAME_QUEUE_BATCH)
+        sqs.delete_queue(QueueUrl=queue_url)
 
 # ---------------
 # HELPER METHODS
@@ -481,19 +575,26 @@ def get_event_source_arn(stream_name):
     return kinesis.describe_stream(StreamName=stream_name)['StreamDescription']['StreamARN']
 
 
-def get_lambda_invocations_count(lambda_name, metric=None):
-    return get_lambda_metrics(lambda_name, metric)['Datapoints'][-1]['Sum']
+def get_lambda_invocations_count(lambda_name, metric=None, period=60, start_time=None, end_time=None):
+    metric = get_lambda_metrics(lambda_name, metric, period, start_time, end_time)
+    if not metric['Datapoints']:
+        return 0
+    return metric['Datapoints'][-1]['Sum']
 
 
-def get_lambda_metrics(func_name, metric=None):
+def get_lambda_metrics(func_name, metric=None, period=60, start_time=None, end_time=None):
     metric = metric or 'Invocations'
     cloudwatch = aws_stack.connect_to_service('cloudwatch')
+    if end_time is None:
+        end_time = datetime.now()
+    if start_time is None:
+        start_time = end_time - timedelta(seconds=period)
     return cloudwatch.get_metric_statistics(
         Namespace='AWS/Lambda',
         MetricName=metric,
         Dimensions=[{'Name': 'FunctionName', 'Value': func_name}],
-        Period=60,
-        StartTime=datetime.now() - timedelta(minutes=1),
-        EndTime=datetime.now(),
+        Period=period,
+        StartTime=start_time,
+        EndTime=end_time,
         Statistics=['Sum']
     )
diff --git a/tests/integration/test_lambda.py b/tests/integration/test_lambda.py
--- a/tests/integration/test_lambda.py
+++ b/tests/integration/test_lambda.py
@@ -27,6 +27,7 @@
 
 THIS_FOLDER = os.path.dirname(os.path.realpath(__file__))
 TEST_LAMBDA_PYTHON = os.path.join(THIS_FOLDER, 'lambdas', 'lambda_integration.py')
+TEST_LAMBDA_PYTHON_ECHO = os.path.join(THIS_FOLDER, 'lambdas', 'lambda_echo.py')
 TEST_LAMBDA_PYTHON3 = os.path.join(THIS_FOLDER, 'lambdas', 'lambda_python3.py')
 TEST_LAMBDA_NODEJS = os.path.join(THIS_FOLDER, 'lambdas', 'lambda_integration.js')
 TEST_LAMBDA_RUBY = os.path.join(THIS_FOLDER, 'lambdas', 'lambda_integration.rb')
diff --git a/tests/integration/test_sqs.py b/tests/integration/test_sqs.py
--- a/tests/integration/test_sqs.py
+++ b/tests/integration/test_sqs.py
@@ -96,6 +96,89 @@ def test_publish_get_delete_message(self):
         # clean up
         self.client.delete_queue(QueueUrl=queue_url)
 
+    def test_publish_get_delete_message_batch(self):
+        queue_name = 'queue-%s' % short_uid()
+        queue_info = self.client.create_queue(QueueName=queue_name)
+        queue_url = queue_info['QueueUrl']
+        self.assertIn(queue_name, queue_url)
+
+        def receive_messages(**kwargs):
+            kwds = dict(
+                QueueUrl=queue_url,
+                MaxNumberOfMessages=10,
+                MessageAttributeNames=['All']
+            )
+            kwds.update(kwargs)
+            messages = self.client.receive_message(**kwds)
+            return messages
+
+        def get_hashes(messages, outgoing=False):
+            body_key = 'MD5OfMessageBody' if outgoing else 'MD5OfBody'
+            return set([
+                (m[body_key], m['MD5OfMessageAttributes'])
+                for m in messages
+            ])
+
+        messages_to_send = [
+            {
+                'Id': 'message{:02d}'.format(i),
+                'MessageBody': 'msgBody{:02d}'.format(i),
+                'MessageAttributes': {
+                    'CustomAttribute': {
+                        'DataType': 'String',
+                        'StringValue': 'CustomAttributeValue{:02d}'.format(i)
+                    }
+                }
+            }
+            for i in range(1, 11)
+        ]
+
+        resp = self.client.send_message_batch(QueueUrl=queue_url, Entries=messages_to_send)
+        sent_hashes = get_hashes(resp.get('Successful', []), outgoing=True)
+        self.assertEqual(len(sent_hashes), len(messages_to_send))
+
+        for i in range(2):
+            messages = receive_messages(VisibilityTimeout=0)['Messages']
+            received_hashes = get_hashes(messages)
+            self.assertEqual(received_hashes, sent_hashes)
+
+        self.client.delete_message_batch(
+            QueueUrl=queue_url,
+            Entries=[
+                {'Id': '{:02d}'.format(i), 'ReceiptHandle': m['ReceiptHandle']}
+                for i, m in enumerate(messages)
+            ]
+        )
+
+        response = receive_messages()
+        self.assertFalse(response.get('Messages'))
+
+        # publish/receive message with change_message_visibility
+        self.client.send_message_batch(QueueUrl=queue_url, Entries=messages_to_send)
+        messages = receive_messages()['Messages']
+        response = receive_messages()
+        self.assertFalse(response.get('Messages'))
+
+        reset_hashes = get_hashes(messages[:5])
+        self.client.change_message_visibility_batch(
+            QueueUrl=queue_url,
+            Entries=[
+                {
+                    'Id': '{:02d}'.format(i),
+                    'ReceiptHandle': msg['ReceiptHandle'],
+                    'VisibilityTimeout': 0,
+                }
+                for i, msg in enumerate(messages[:5])
+            ]
+        )
+        for i in range(2):
+            messages = receive_messages(VisibilityTimeout=0)['Messages']
+            received_hashes = get_hashes(messages)
+            self.assertEqual(reset_hashes, received_hashes)
+
+        # clean up
+        self.client.delete_queue(QueueUrl=queue_url)
+
     def test_create_fifo_queue(self):
         fifo_queue = 'my-queue.fifo'
         queue_info = self.client.create_queue(QueueName=fifo_queue, Attributes={'FifoQueue': 'true'})