Repository: localstack/localstack

Issue:
Lambda event body illegally quoted
When quoting the event body of a lambda event, quotes are [first escaped manually](https://github.com/localstack/localstack/blob/6ff360533f66a170dcb4b4314d4e729236de24e5/localstack/services/awslambda/lambda_executors.py#L168), [then escaped by shlex](https://github.com/localstack/localstack/blob/6ff360533f66a170dcb4b4314d4e729236de24e5/localstack/services/awslambda/lambda_executors.py#L286) (or pipes is python 2.7).

This first pass of escapes produces illegal JSON. To be precise, it produces JSON that is load-able by Python, but unmarshal-able by Go, which does not include `\'` in its list of valid esscape sequences (per the JSON spec).

To get around this issue, we currently need to ensure that none of our values used for local testing include a single quote character, otherwise our DynamoDB streams consumers will be _poisoned_ until a forceful restart and purge of the data. For a complete report, our troubles end [here](https://github.com/aws/aws-lambda-go/blob/master/lambda/handler.go#L113).

To illustrate this issue we need to first generate the escaped event the same way it is generated in localstack:

```python
>>> import json, shlex
>>> print(shlex.quote(json.dumps({'test': 'it\'s illegal'}).replace("'", "\\'")))
'{"test": "it\'"'"'s illegal"}'
```

Now, if we test to see how this is interpreted in the shell, we get something interesting.

```bash
$ export TEST='{"test": "it\'"'"'s illegal"}' && env | grep TEST
TEST={"test": "it\'s illegal"}
```

This seems like valid JSON, but the `\'`is an illegal escape and cannot be decoded universally. The solution may be as simple as removing the superfluous `string.replace` call indicated above.


Discussion:

Resolving patch:
diff --git a/localstack/services/dynamodb/dynamodb_listener.py b/localstack/services/dynamodb/dynamodb_listener.py
--- a/localstack/services/dynamodb/dynamodb_listener.py
+++ b/localstack/services/dynamodb/dynamodb_listener.py
@@ -25,7 +25,6 @@
 
 
 class ProxyListenerDynamoDB(ProxyListener):
-
     thread_local = threading.local()
 
     def __init__(self):
@@ -44,6 +43,23 @@ def forward_request(self, method, path, data, headers):
             # find an existing item and store it in a thread-local, so we can access it in return_response,
             # in order to determine whether an item already existed (MODIFY) or not (INSERT)
             ProxyListenerDynamoDB.thread_local.existing_item = find_existing_item(data)
+        elif action == '%s.BatchWriteItem' % ACTION_PREFIX:
+            existing_items = []
+            for table_name in sorted(data['RequestItems'].keys()):
+                for request in data['RequestItems'][table_name]:
+                    for key in ['PutRequest', 'DeleteRequest']:
+                        inner_request = request.get(key)
+                        if inner_request:
+                            existing_items.append(find_existing_item(inner_request, table_name))
+            ProxyListenerDynamoDB.thread_local.existing_items = existing_items
+        elif action == '%s.TransactWriteItems' % ACTION_PREFIX:
+            existing_items = []
+            for item in data['TransactItems']:
+                for key in ['Put', 'Update', 'Delete']:
+                    inner_item = item.get(key)
+                    if inner_item:
+                        existing_items.append(find_existing_item(inner_item))
+            ProxyListenerDynamoDB.thread_local.existing_items = existing_items
         elif action == '%s.UpdateTimeToLive' % ACTION_PREFIX:
             # TODO: TTL status is maintained/mocked but no real expiry is happening for items
             response = Response()
@@ -132,24 +148,12 @@ def return_response(self, method, path, data, headers, response):
                 record['dynamodb']['NewImage'] = updated_item
                 record['dynamodb']['SizeBytes'] = len(json.dumps(updated_item))
         elif action == '%s.BatchWriteItem' % ACTION_PREFIX:
-            records = []
-            for table_name, requests in data['RequestItems'].items():
-                for request in requests:
-                    put_request = request.get('PutRequest')
-                    if put_request:
-                        keys = dynamodb_extract_keys(item=put_request['Item'], table_name=table_name)
-                        if isinstance(keys, Response):
-                            return keys
-                        new_record = clone(record)
-                        new_record['eventName'] = 'INSERT'
-                        new_record['dynamodb']['Keys'] = keys
-                        new_record['dynamodb']['NewImage'] = put_request['Item']
-                        new_record['eventSourceARN'] = aws_stack.dynamodb_table_arn(table_name)
-                        records.append(new_record)
+            records = self.prepare_batch_write_item_records(record, data)
+        elif action == '%s.TransactWriteItems' % ACTION_PREFIX:
+            records = self.prepare_transact_write_item_records(record, data)
         elif action == '%s.PutItem' % ACTION_PREFIX:
             if response.status_code == 200:
                 existing_item = self._thread_local('existing_item')
-                ProxyListenerDynamoDB.thread_local.existing_item = None
                 record['eventName'] = 'INSERT' if not existing_item else 'MODIFY'
                 keys = dynamodb_extract_keys(item=data['Item'], table_name=data['TableName'])
                 if isinstance(keys, Response):
@@ -201,19 +205,100 @@ def return_response(self, method, path, data, headers, response):
             forward_to_lambda(records)
             forward_to_ddb_stream(records)
 
-    def _thread_local(self, name):
+    def prepare_batch_write_item_records(self, record, data):
+        records = []
+        i = 0
+        for table_name in sorted(data['RequestItems'].keys()):
+            for request in data['RequestItems'][table_name]:
+                put_request = request.get('PutRequest')
+                if put_request:
+                    existing_item = self._thread_local('existing_items')[i]
+                    keys = dynamodb_extract_keys(item=put_request['Item'], table_name=table_name)
+                    if isinstance(keys, Response):
+                        return keys
+                    new_record = clone(record)
+                    new_record['eventName'] = 'INSERT' if not existing_item else 'MODIFY'
+                    new_record['dynamodb']['Keys'] = keys
+                    new_record['dynamodb']['NewImage'] = put_request['Item']
+                    if existing_item:
+                        new_record['dynamodb']['OldImage'] = existing_item
+                    new_record['eventSourceARN'] = aws_stack.dynamodb_table_arn(table_name)
+                    records.append(new_record)
+                delete_request = request.get('DeleteRequest')
+                if delete_request:
+                    keys = delete_request['Key']
+                    if isinstance(keys, Response):
+                        return keys
+                    new_record = clone(record)
+                    new_record['eventName'] = 'REMOVE'
+                    new_record['dynamodb']['Keys'] = keys
+                    new_record['dynamodb']['OldImage'] = self._thread_local('existing_items')[i]
+                    new_record['eventSourceARN'] = aws_stack.dynamodb_table_arn(table_name)
+                    records.append(new_record)
+                i += 1
+        return records
+
+    def prepare_transact_write_item_records(self, record, data):
+        records = []
+        for i, request in enumerate(data['TransactItems']):
+            put_request = request.get('Put')
+            if put_request:
+                existing_item = self._thread_local('existing_items')[i]
+                table_name = put_request['TableName']
+                keys = dynamodb_extract_keys(item=put_request['Item'], table_name=table_name)
+                if isinstance(keys, Response):
+                    return keys
+                new_record = clone(record)
+                new_record['eventName'] = 'INSERT' if not existing_item else 'MODIFY'
+                new_record['dynamodb']['Keys'] = keys
+                new_record['dynamodb']['NewImage'] = put_request['Item']
+                if existing_item:
+                    new_record['dynamodb']['OldImage'] = existing_item
+                new_record['eventSourceARN'] = aws_stack.dynamodb_table_arn(table_name)
+                records.append(new_record)
+            update_request = request.get('Update')
+            if update_request:
+                table_name = update_request['TableName']
+                keys = update_request['Key']
+                if isinstance(keys, Response):
+                    return keys
+                updated_item = find_existing_item(update_request, table_name)
+                if not updated_item:
+                    return
+                new_record = clone(record)
+                new_record['eventName'] = 'MODIFY'
+                new_record['dynamodb']['Keys'] = keys
+                new_record['dynamodb']['OldImage'] = self._thread_local('existing_items')[i]
+                new_record['dynamodb']['NewImage'] = updated_item
+                new_record['eventSourceARN'] = aws_stack.dynamodb_table_arn(table_name)
+                records.append(new_record)
+            delete_request = request.get('Delete')
+            if delete_request:
+                table_name = delete_request['TableName']
+                keys = delete_request['Key']
+                if isinstance(keys, Response):
+                    return keys
+                new_record = clone(record)
+                new_record['eventName'] = 'REMOVE'
+                new_record['dynamodb']['Keys'] = keys
+                new_record['dynamodb']['OldImage'] = self._thread_local('existing_items')[i]
+                new_record['eventSourceARN'] = aws_stack.dynamodb_table_arn(table_name)
+                records.append(new_record)
+        return records
+
+    def _thread_local(self, name, default=None):
         try:
             return getattr(ProxyListenerDynamoDB.thread_local, name)
         except AttributeError:
-            return None
+            return default
 
 
 # instantiate listener
 UPDATE_DYNAMODB = ProxyListenerDynamoDB()
 
 
-def find_existing_item(put_item):
-    table_name = put_item['TableName']
+def find_existing_item(put_item, table_name=None):
+    table_name = table_name or put_item['TableName']
     ddb_client = aws_stack.connect_to_service('dynamodb')
 
     search_key = {}
diff --git a/localstack/utils/testutil.py b/localstack/utils/testutil.py
--- a/localstack/utils/testutil.py
+++ b/localstack/utils/testutil.py
@@ -77,10 +77,18 @@ def create_zip_file(file_path, get_content=False):
 
 def create_lambda_function(func_name, zip_file, event_source_arn=None, handler=LAMBDA_DEFAULT_HANDLER,
         starting_position=LAMBDA_DEFAULT_STARTING_POSITION, runtime=LAMBDA_DEFAULT_RUNTIME,
-        envvars={}, tags={}):
+        envvars={}, tags={}, delete=False):
     """Utility method to create a new function via the Lambda API"""
 
     client = aws_stack.connect_to_service('lambda')
+
+    if delete:
+        try:
+            # Delete function if one already exists
+            client.delete_function(FunctionName=func_name)
+        except Exception:
+            pass
+
     # create function
     client.create_function(
         FunctionName=func_name,


Test patch:
diff --git a/tests/integration/lambdas/lambda_integration.py b/tests/integration/lambdas/lambda_integration.py
--- a/tests/integration/lambdas/lambda_integration.py
+++ b/tests/integration/lambdas/lambda_integration.py
@@ -96,9 +96,16 @@ def deserialize_event(event):
     # Deserialize into Python dictionary and extract the "NewImage" (the new version of the full ddb document)
     ddb = event.get('dynamodb')
     if ddb:
+        result = {
+            '__action_type': event.get('eventName'),
+        }
+
         ddb_deserializer = TypeDeserializer()
-        result = ddb_deserializer.deserialize({'M': ddb.get('NewImage')})
-        result['__action_type'] = event.get('eventName')
+        if ddb.get('OldImage'):
+            result['old_image'] = ddb_deserializer.deserialize({'M': ddb.get('OldImage')})
+        if ddb.get('NewImage'):
+            result['new_image'] = ddb_deserializer.deserialize({'M': ddb.get('NewImage')})
+
         return result
     kinesis = event.get('kinesis')
     if kinesis:
diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py
--- a/tests/integration/test_integration.py
+++ b/tests/integration/test_integration.py
@@ -32,8 +32,6 @@
 TEST_CHAIN_LAMBDA1_NAME = 'test_chain_lambda_1'
 TEST_CHAIN_LAMBDA2_NAME = 'test_chain_lambda_2'
 
-EVENTS = []
-
 PARTITION_KEY = 'id'
 
 # set up logger
@@ -122,6 +120,8 @@ def test_firehose_kinesis_to_s3(self):
 
     def test_kinesis_lambda_sns_ddb_sqs_streams(self):
         ddb_lease_table_suffix = '-kclapp'
+        table_name = TEST_TABLE_NAME + 'klsdss' + ddb_lease_table_suffix
+        stream_name = TEST_STREAM_NAME
         dynamodb = aws_stack.connect_to_resource('dynamodb')
         dynamodb_service = aws_stack.connect_to_service('dynamodb')
         dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
@@ -131,29 +131,31 @@ def test_kinesis_lambda_sns_ddb_sqs_streams(self):
 
         LOGGER.info('Creating test streams...')
         run_safe(lambda: dynamodb_service.delete_table(
-            TableName=TEST_STREAM_NAME + ddb_lease_table_suffix), print_error=False)
-        aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True)
+            TableName=stream_name + ddb_lease_table_suffix), print_error=False)
+        aws_stack.create_kinesis_stream(stream_name, delete=True)
         aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
 
+        events = []
+
         # subscribe to inbound Kinesis stream
         def process_records(records, shard_id):
-            EVENTS.extend(records)
+            events.extend(records)
 
         # start the KCL client process in the background
-        kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records,
+        kinesis_connector.listen_to_kinesis(stream_name, listener_func=process_records,
             wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix)
 
         LOGGER.info('Kinesis consumer initialized.')
 
         # create table with stream forwarding config
-        aws_stack.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY,
+        aws_stack.create_dynamodb_table(table_name, partition_key=PARTITION_KEY,
             stream_view_type='NEW_AND_OLD_IMAGES')
 
         # list DDB streams and make sure the table stream is there
         streams = dynamodbstreams.list_streams()
         ddb_event_source_arn = None
         for stream in streams['Streams']:
-            if stream['TableName'] == TEST_TABLE_NAME:
+            if stream['TableName'] == table_name:
                 ddb_event_source_arn = stream['StreamArn']
         self.assertTrue(ddb_event_source_arn)
 
@@ -161,7 +163,7 @@ def process_records(records, shard_id):
         zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True,
             libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27)
         testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
-            zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
+            zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27, delete=True)
         # make sure we cannot create Lambda with same name twice
         assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB,
             zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
@@ -185,7 +187,7 @@ def process_records(records, shard_id):
         num_updates_ddb = num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items
 
         LOGGER.info('Putting %s items to table...' % num_events_ddb)
-        table = dynamodb.Table(TEST_TABLE_NAME)
+        table = dynamodb.Table(table_name)
         for i in range(0, num_put_new_items):
             table.put_item(Item={
                 PARTITION_KEY: 'testId%s' % i,
@@ -199,14 +201,14 @@ def process_records(records, shard_id):
             })
 
         # batch write some items containing non-ASCII characters
-        dynamodb.batch_write_item(RequestItems={TEST_TABLE_NAME: [
+        dynamodb.batch_write_item(RequestItems={table_name: [
             {'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ✓'}}},
             {'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 £'}}},
             {'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ¢'}}}
         ]})
         # update some items, which also triggers notification events
         for i in range(0, num_updates_ddb):
-            dynamodb_service.update_item(TableName=TEST_TABLE_NAME,
+            dynamodb_service.update_item(TableName=table_name,
                 Key={PARTITION_KEY: {'S': 'testId%s' % i}},
                 AttributeUpdates={'data': {
                     'Action': 'PUT',
@@ -254,11 +256,11 @@ def process_records(records, shard_id):
         num_events = num_events_lambda + num_events_kinesis
 
         def check_events():
-            if len(EVENTS) != num_events:
+            if len(events) != num_events:
                 LOGGER.warning(('DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s') %
-                    (len(EVENTS), num_events))
-            self.assertEqual(len(EVENTS), num_events)
-            event_items = [json.loads(base64.b64decode(e['data'])) for e in EVENTS]
+                    (len(events), num_events))
+            self.assertEqual(len(events), num_events)
+            event_items = [json.loads(base64.b64decode(e['data'])) for e in events]
             # make sure the we have the right amount of INSERT/MODIFY event types
             inserts = [e for e in event_items if e.get('__action_type') == 'INSERT']
             modifies = [e for e in event_items if e.get('__action_type') == 'MODIFY']
@@ -280,6 +282,151 @@ def check_events():
         num_error_invocations = get_lambda_invocations_count(TEST_LAMBDA_NAME_STREAM, 'Errors')
         self.assertEqual(num_error_invocations, 1)
 
+    def test_lambda_streams_batch_and_transactions(self):
+        ddb_lease_table_suffix = '-kclapp2'
+        table_name = TEST_TABLE_NAME + 'lsbat' + ddb_lease_table_suffix
+        stream_name = TEST_STREAM_NAME
+        dynamodb = aws_stack.connect_to_service('dynamodb', client=True)
+        dynamodb_service = aws_stack.connect_to_service('dynamodb')
+        dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
+
+        LOGGER.info('Creating test streams...')
+        run_safe(lambda: dynamodb_service.delete_table(
+            TableName=stream_name + ddb_lease_table_suffix), print_error=False)
+        aws_stack.create_kinesis_stream(stream_name, delete=True)
+
+        events = []
+
+        # subscribe to inbound Kinesis stream
+        def process_records(records, shard_id):
+            events.extend(records)
+
+        # start the KCL client process in the background
+        kinesis_connector.listen_to_kinesis(stream_name, listener_func=process_records,
+            wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix)
+
+        LOGGER.info('Kinesis consumer initialized.')
+
+        # create table with stream forwarding config
+        aws_stack.create_dynamodb_table(table_name, partition_key=PARTITION_KEY,
+            stream_view_type='NEW_AND_OLD_IMAGES')
+
+        # list DDB streams and make sure the table stream is there
+        streams = dynamodbstreams.list_streams()
+        ddb_event_source_arn = None
+        for stream in streams['Streams']:
+            if stream['TableName'] == table_name:
+                ddb_event_source_arn = stream['StreamArn']
+        self.assertTrue(ddb_event_source_arn)
+
+        # deploy test lambda connected to DynamoDB Stream
+        zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True,
+            libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27)
+        testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
+            zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27, delete=True)
+
+        # submit a batch with writes
+        dynamodb.batch_write_item(RequestItems={table_name: [
+            {'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId0'}, 'data': {'S': 'foobar123'}}}},
+            {'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId1'}, 'data': {'S': 'foobar123'}}}},
+            {'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId2'}, 'data': {'S': 'foobar123'}}}}
+        ]})
+
+        # submit a batch with writes and deletes
+        dynamodb.batch_write_item(RequestItems={table_name: [
+            {'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId3'}, 'data': {'S': 'foobar123'}}}},
+            {'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId4'}, 'data': {'S': 'foobar123'}}}},
+            {'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId5'}, 'data': {'S': 'foobar123'}}}},
+            {'DeleteRequest': {'Key': {PARTITION_KEY: {'S': 'testId0'}}}},
+            {'DeleteRequest': {'Key': {PARTITION_KEY: {'S': 'testId1'}}}},
+            {'DeleteRequest': {'Key': {PARTITION_KEY: {'S': 'testId2'}}}},
+        ]})
+
+        # submit a transaction with writes and delete
+        dynamodb.transact_write_items(TransactItems=[
+            {'Put': {'TableName': table_name,
+                'Item': {PARTITION_KEY: {'S': 'testId6'}, 'data': {'S': 'foobar123'}}}},
+            {'Put': {'TableName': table_name,
+                'Item': {PARTITION_KEY: {'S': 'testId7'}, 'data': {'S': 'foobar123'}}}},
+            {'Put': {'TableName': table_name,
+                'Item': {PARTITION_KEY: {'S': 'testId8'}, 'data': {'S': 'foobar123'}}}},
+            {'Delete': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId3'}}}},
+            {'Delete': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId4'}}}},
+            {'Delete': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId5'}}}},
+        ])
+
+        # submit a batch with a put over existing item
+        dynamodb.transact_write_items(TransactItems=[
+            {'Put': {'TableName': table_name,
+                'Item': {PARTITION_KEY: {'S': 'testId6'}, 'data': {'S': 'foobar123_updated1'}}}},
+        ])
+
+        # submit a transaction with a put over existing item
+        dynamodb.transact_write_items(TransactItems=[
+            {'Put': {'TableName': table_name,
+                'Item': {PARTITION_KEY: {'S': 'testId7'}, 'data': {'S': 'foobar123_updated1'}}}},
+        ])
+
+        # submit a transaction with updates
+        dynamodb.transact_write_items(TransactItems=[
+            {'Update': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId6'}},
+                'UpdateExpression': 'SET #0 = :0',
+                'ExpressionAttributeNames': {'#0': 'data'},
+                'ExpressionAttributeValues': {':0': {'S': 'foobar123_updated2'}}}},
+            {'Update': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId7'}},
+                'UpdateExpression': 'SET #0 = :0',
+                'ExpressionAttributeNames': {'#0': 'data'},
+                'ExpressionAttributeValues': {':0': {'S': 'foobar123_updated2'}}}},
+            {'Update': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId8'}},
+                'UpdateExpression': 'SET #0 = :0',
+                'ExpressionAttributeNames': {'#0': 'data'},
+                'ExpressionAttributeValues': {':0': {'S': 'foobar123_updated2'}}}},
+        ])
+
+        LOGGER.info('Waiting some time before finishing test.')
+        time.sleep(2)
+
+        num_insert = 9
+        num_modify = 5
+        num_delete = 6
+        num_events = num_insert + num_modify + num_delete
+
+        def check_events():
+            if len(events) != num_events:
+                LOGGER.warning(('DynamoDB updates retrieved (actual/expected): %s/%s') %
+                    (len(events), num_events))
+            self.assertEqual(len(events), num_events)
+            event_items = [json.loads(base64.b64decode(e['data'])) for e in events]
+            # make sure the we have the right amount of expected event types
+            inserts = [e for e in event_items if e.get('__action_type') == 'INSERT']
+            modifies = [e for e in event_items if e.get('__action_type') == 'MODIFY']
+            removes = [e for e in event_items if e.get('__action_type') == 'REMOVE']
+            self.assertEqual(len(inserts), num_insert)
+            self.assertEqual(len(modifies), num_modify)
+            self.assertEqual(len(removes), num_delete)
+
+            for i, event in enumerate(inserts):
+                self.assertNotIn('old_image', event)
+                self.assertEqual(inserts[i]['new_image'], {'id': 'testId%d' % i, 'data': 'foobar123'})
+
+            self.assertEqual(modifies[0]['old_image'], {'id': 'testId6', 'data': 'foobar123'})
+            self.assertEqual(modifies[0]['new_image'], {'id': 'testId6', 'data': 'foobar123_updated1'})
+            self.assertEqual(modifies[1]['old_image'], {'id': 'testId7', 'data': 'foobar123'})
+            self.assertEqual(modifies[1]['new_image'], {'id': 'testId7', 'data': 'foobar123_updated1'})
+            self.assertEqual(modifies[2]['old_image'], {'id': 'testId6', 'data': 'foobar123_updated1'})
+            self.assertEqual(modifies[2]['new_image'], {'id': 'testId6', 'data': 'foobar123_updated2'})
+            self.assertEqual(modifies[3]['old_image'], {'id': 'testId7', 'data': 'foobar123_updated1'})
+            self.assertEqual(modifies[3]['new_image'], {'id': 'testId7', 'data': 'foobar123_updated2'})
+            self.assertEqual(modifies[4]['old_image'], {'id': 'testId8', 'data': 'foobar123'})
+            self.assertEqual(modifies[4]['new_image'], {'id': 'testId8', 'data': 'foobar123_updated2'})
+
+            for i, event in enumerate(removes):
+                self.assertEqual(event['old_image'], {'id': 'testId%d' % i, 'data': 'foobar123'})
+                self.assertNotIn('new_image', event)
+
+        # this can take a long time in CI, make sure we give it enough time/retries
+        retry(check_events, retries=9, sleep=3)
+
     def test_kinesis_lambda_forward_chain(self):
         kinesis = aws_stack.connect_to_service('kinesis')
         s3 = aws_stack.connect_to_service('s3')