Repository: localstack/localstack

Issue:
Dynamodb BatchWriteItem not Throttling Properly
<!-- Love localstack? Please consider supporting our collective:
👉  https://opencollective.com/localstack/donate -->

# Type of request: This is a ...

[ X ] bug report
[ ] feature request

# Detailed description

On BatchWriteItem if items are throttled, we are expected to receive a list of `UnprocessedItems` in the response instead of HTTP 500

## Expected behavior

Receive HTTP 200 but with Unprocessed Items in the response: https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html#DDB-BatchWriteItem-response-UnprocessedItems

## Actual behavior

Receive HTTP 500 ProvisionedThroughPutException. "All or nothing"

# Steps to reproduce

Start LocalStack with DynamoDb Service and "DYNAMODB_ERROR_PROBABILITY=1.0"

## Command used to start LocalStack

```
new LocalStackContainer(DockerImageName.parse("localstack/localstack:0.12.9"))
      .withEnv("DYNAMODB_ERROR_PROBABILITY", "1.0")
      .withServices(DYNAMODB)
```

## Client code (AWS SDK code snippet, or sequence of "awslocal" commands)

You can just Randomly generate a big list of objects (exceeding 1kb per batch) and check this behaviour




Discussion:
Update: Just lowered the probability to 0.2 and i can confirm that there's no partial failures for the BatchWriteItem requests. Current behaviour is "All or nothing"

Resolving patch:
diff --git a/localstack/config.py b/localstack/config.py
--- a/localstack/config.py
+++ b/localstack/config.py
@@ -221,7 +221,8 @@ def is_linux():
 CONFIG_ENV_VARS = ['SERVICES', 'HOSTNAME', 'HOSTNAME_EXTERNAL', 'LOCALSTACK_HOSTNAME', 'LAMBDA_FALLBACK_URL',
                    'LAMBDA_EXECUTOR', 'LAMBDA_REMOTE_DOCKER', 'LAMBDA_DOCKER_NETWORK', 'LAMBDA_REMOVE_CONTAINERS',
                    'USE_SSL', 'DEBUG', 'KINESIS_ERROR_PROBABILITY', 'DYNAMODB_ERROR_PROBABILITY', 'PORT_WEB_UI',
-                   'START_WEB', 'DOCKER_BRIDGE_IP', 'DEFAULT_REGION', 'LAMBDA_JAVA_OPTS', 'LOCALSTACK_API_KEY',
+                   'DYNAMODB_READ_ERROR_PROBABILITY', 'DYNAMODB_WRITE_ERROR_PROBABILITY', 'START_WEB',
+                   'DOCKER_BRIDGE_IP', 'DEFAULT_REGION', 'LAMBDA_JAVA_OPTS', 'LOCALSTACK_API_KEY',
                    'LAMBDA_CONTAINER_REGISTRY', 'TEST_AWS_ACCOUNT_ID', 'DISABLE_EVENTS', 'EDGE_PORT', 'LS_LOG',
                    'EDGE_PORT_HTTP', 'SKIP_INFRA_DOWNLOADS', 'STEPFUNCTIONS_LAMBDA_ENDPOINT',
                    'WINDOWS_DOCKER_MOUNT_PREFIX', 'HOSTNAME_FROM_LAMBDA',
diff --git a/localstack/services/dynamodb/dynamodb_listener.py b/localstack/services/dynamodb/dynamodb_listener.py
--- a/localstack/services/dynamodb/dynamodb_listener.py
+++ b/localstack/services/dynamodb/dynamodb_listener.py
@@ -67,7 +67,7 @@ def table_exists(ddb_client, table_name):
 
     def action_should_throttle(self, action, actions):
         throttled = ['%s%s' % (ACTION_PREFIX, a) for a in actions]
-        return action in throttled
+        return (action in throttled) or (action in actions)
 
     def should_throttle(self, action):
         rand = random.random()
@@ -97,6 +97,7 @@ def forward_request(self, method, path, data, headers):
         ddb_client = aws_stack.connect_to_service('dynamodb')
         action = headers.get('X-Amz-Target', '')
         action = action.replace(ACTION_PREFIX, '')
+
         if self.should_throttle(action):
             return error_response_throughput()
 
@@ -152,13 +153,24 @@ def forward_request(self, method, path, data, headers):
 
         elif action == 'BatchWriteItem':
             existing_items = []
+            unprocessed_put_items = []
+            unprocessed_delete_items = []
             for table_name in sorted(data['RequestItems'].keys()):
                 for request in data['RequestItems'][table_name]:
                     for key in ['PutRequest', 'DeleteRequest']:
                         inner_request = request.get(key)
                         if inner_request:
-                            existing_items.append(find_existing_item(inner_request, table_name))
+                            if self.should_throttle(action):
+                                if key == 'PutRequest':
+                                    unprocessed_put_items.append(inner_request)
+                                elif key == 'DeleteRequest':
+                                    unprocessed_delete_items.append(inner_request)
+                            else:
+                                item = find_existing_item(inner_request, table_name)
+                                existing_items.append(item)
             ProxyListenerDynamoDB.thread_local.existing_items = existing_items
+            ProxyListenerDynamoDB.thread_local.unprocessed_put_items = unprocessed_put_items
+            ProxyListenerDynamoDB.thread_local.unprocessed_delete_items = unprocessed_delete_items
 
         elif action == 'Query':
             if data.get('IndexName'):
@@ -303,6 +315,7 @@ def return_response(self, method, path, data, headers, response):
         streams_enabled_cache = {}
         table_name = data.get('TableName')
         event_sources_or_streams_enabled = has_event_sources_or_streams_enabled(table_name, streams_enabled_cache)
+
         if action == 'UpdateItem':
             if response.status_code == 200 and event_sources_or_streams_enabled:
                 existing_item = self._thread_local('existing_item')
@@ -319,11 +332,22 @@ def return_response(self, method, path, data, headers, response):
                 stream_spec = dynamodb_get_table_stream_specification(table_name=table_name)
                 if stream_spec:
                     record['dynamodb']['StreamViewType'] = stream_spec['StreamViewType']
+
         elif action == 'BatchWriteItem':
-            records = self.prepare_batch_write_item_records(record, data)
+            records, unprocessed_items = self.prepare_batch_write_item_records(record, data)
             for record in records:
                 event_sources_or_streams_enabled = (event_sources_or_streams_enabled or
                     has_event_sources_or_streams_enabled(record['eventSourceARN'], streams_enabled_cache))
+            if response.status_code == 200 and any(unprocessed_items):
+                content = json.loads(to_str(response.content))
+                table_name = list(data['RequestItems'].keys())[0]
+                if table_name not in content['UnprocessedItems']:
+                    content['UnprocessedItems'][table_name] = []
+                for key in ['PutRequest', 'DeleteRequest']:
+                    if any(unprocessed_items[key]):
+                        content['UnprocessedItems'][table_name].append({key: unprocessed_items[key]})
+                response._content = json.dumps(content)
+                fix_headers_for_updated_response(response)
 
         elif action == 'TransactWriteItems':
             records = self.prepare_transact_write_item_records(record, data)
@@ -455,6 +479,7 @@ def prepare_request_headers(self, headers):
 
     def prepare_batch_write_item_records(self, record, data):
         records = []
+        unprocessed_items = {'PutRequest': {}, 'DeleteRequest': {}}
         i = 0
         for table_name in sorted(data['RequestItems'].keys()):
             # Add stream view type to record if ddb stream is enabled
@@ -463,37 +488,49 @@ def prepare_batch_write_item_records(self, record, data):
                 record['dynamodb']['StreamViewType'] = stream_spec['StreamViewType']
             for request in data['RequestItems'][table_name]:
                 put_request = request.get('PutRequest')
+                existing_items = self._thread_local('existing_items')
                 if put_request:
-                    existing_item = self._thread_local('existing_items')[i]
-                    keys = dynamodb_extract_keys(item=put_request['Item'], table_name=table_name)
-                    if isinstance(keys, Response):
-                        return keys
-                    new_record = clone(record)
-                    new_record['eventID'] = short_uid()
-                    new_record['dynamodb']['SizeBytes'] = len(json.dumps(put_request['Item']))
-                    new_record['eventName'] = 'INSERT' if not existing_item else 'MODIFY'
-                    new_record['dynamodb']['Keys'] = keys
-                    new_record['dynamodb']['NewImage'] = put_request['Item']
-                    if existing_item:
-                        new_record['dynamodb']['OldImage'] = existing_item
-                    new_record['eventSourceARN'] = aws_stack.dynamodb_table_arn(table_name)
-                    records.append(new_record)
+                    if existing_items and len(existing_items) > i:
+                        existing_item = existing_items[i]
+                        keys = dynamodb_extract_keys(item=put_request['Item'], table_name=table_name)
+                        if isinstance(keys, Response):
+                            return keys
+                        new_record = clone(record)
+                        new_record['eventID'] = short_uid()
+                        new_record['dynamodb']['SizeBytes'] = len(json.dumps(put_request['Item']))
+                        new_record['eventName'] = 'INSERT' if not existing_item else 'MODIFY'
+                        new_record['dynamodb']['Keys'] = keys
+                        new_record['dynamodb']['NewImage'] = put_request['Item']
+                        if existing_item:
+                            new_record['dynamodb']['OldImage'] = existing_item
+                        new_record['eventSourceARN'] = aws_stack.dynamodb_table_arn(table_name)
+                        records.append(new_record)
+                    unprocessed_put_items = self._thread_local('unprocessed_put_items')
+                    if unprocessed_put_items and len(unprocessed_put_items) > i:
+                        unprocessed_item = unprocessed_put_items[i]
+                        if unprocessed_item:
+                            unprocessed_items['PutRequest'].update(json.loads(json.dumps(unprocessed_item)))
                 delete_request = request.get('DeleteRequest')
                 if delete_request:
-                    keys = delete_request['Key']
-                    existing_item = self._thread_local('existing_items')[i]
-                    if isinstance(keys, Response):
-                        return keys
-                    new_record = clone(record)
-                    new_record['eventID'] = short_uid()
-                    new_record['eventName'] = 'REMOVE'
-                    new_record['dynamodb']['Keys'] = keys
-                    new_record['dynamodb']['OldImage'] = existing_item
-                    new_record['dynamodb']['SizeBytes'] = len(json.dumps(existing_item))
-                    new_record['eventSourceARN'] = aws_stack.dynamodb_table_arn(table_name)
-                    records.append(new_record)
+                    if existing_items and len(existing_items) > i:
+                        keys = delete_request['Key']
+                        if isinstance(keys, Response):
+                            return keys
+                        new_record = clone(record)
+                        new_record['eventID'] = short_uid()
+                        new_record['eventName'] = 'REMOVE'
+                        new_record['dynamodb']['Keys'] = keys
+                        new_record['dynamodb']['OldImage'] = existing_items[i]
+                        new_record['dynamodb']['SizeBytes'] = len(json.dumps(existing_item))
+                        new_record['eventSourceARN'] = aws_stack.dynamodb_table_arn(table_name)
+                        records.append(new_record)
+                    unprocessed_delete_items = self._thread_local('unprocessed_delete_items')
+                    if unprocessed_delete_items and len(unprocessed_delete_items) > i:
+                        unprocessed_item = unprocessed_delete_items[i]
+                        if unprocessed_item:
+                            unprocessed_items['DeleteRequest'].update(json.loads(json.dumps(unprocessed_item)))
                 i += 1
-        return records
+        return records, unprocessed_items
 
     def prepare_transact_write_item_records(self, record, data):
         records = []


Test patch:
diff --git a/tests/integration/test_error_injection.py b/tests/integration/test_error_injection.py
--- a/tests/integration/test_error_injection.py
+++ b/tests/integration/test_error_injection.py
@@ -99,6 +99,10 @@ def test_dynamodb_write_error_injection():
     config.DYNAMODB_WRITE_ERROR_PROBABILITY = 1.0
     assert_raises(ClientError, table.put_item, Item={PARTITION_KEY: short_uid(), 'data': 'foobar123'})
 
+    # BatchWriteItem throws ProvisionedThroughputExceededException if ALL items in Batch are Throttled
+    assert_raises(ClientError, table.batch_write_item, RequestItems={table: [{
+        'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123'}}}]})
+
     # reset probability to zero
     config.DYNAMODB_WRITE_ERROR_PROBABILITY = 0.0