Repository: boto/boto

Issue:
boto 2.10.0 dynamodb2 batch_write failed
I have encountered the problem, when the write speed exceeded the throughtput, the batch_write would silently failed without any Exception. I have check the source code in boto.dynamodb2.table.Table, and find the problem. Boto has not check the response from aws dynamodb api, and ignored the response. 



Discussion:
Sliently failed when the throughtput had bean exceeded.

Sorry you ran into this.

Unfortunately, the DDB API doesn't give a concrete error in this case. It will return "UnprocessedItems" (http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html, second paragraph), but we have no indication whether it's due to error or just exceeded throughput. It's on the TODO list to implement a fix, but we need to figure how (& how many times) we'll retry items.

@toastdriven  isn't this duplicate of the same bug like here https://github.com/boto/boto/issues/1566?

Yes, it is.

@toastdriven could you (or could I help with that, I'm not an expert with github) label proper tickets related to proper funcionality? I'm looking into this tickets and maybe I will try to help with them but not every dynamodb is labeled and some as this above are duplicates :) it would greatly help to developers that are interested in only some parts of boto (for example only dynamodb or only ec2. )

Cheers,
Kamil 


Resolving patch:
diff --git a/boto/dynamodb2/layer1.py b/boto/dynamodb2/layer1.py
--- a/boto/dynamodb2/layer1.py
+++ b/boto/dynamodb2/layer1.py
@@ -1491,6 +1491,7 @@ def make_request(self, action, body):
 
     def _retry_handler(self, response, i, next_sleep):
         status = None
+        boto.log.debug("Saw HTTP status: %s" % response.status)
         if response.status == 400:
             response_body = response.read()
             boto.log.debug(response_body)
diff --git a/boto/dynamodb2/table.py b/boto/dynamodb2/table.py
--- a/boto/dynamodb2/table.py
+++ b/boto/dynamodb2/table.py
@@ -1,3 +1,4 @@
+import boto
 from boto.dynamodb2 import exceptions
 from boto.dynamodb2.fields import (HashKey, RangeKey,
                                    AllIndex, KeysOnlyIndex, IncludeIndex)
@@ -1070,17 +1071,19 @@ def __init__(self, table):
         self.table = table
         self._to_put = []
         self._to_delete = []
+        self._unprocessed = []
 
     def __enter__(self):
         return self
 
     def __exit__(self, type, value, traceback):
-        if not self._to_put and not self._to_delete:
-            return False
+        if self._to_put or self._to_delete:
+            # Flush anything that's left.
+            self.flush()
 
-        # Flush anything that's left.
-        self.flush()
-        return True
+        if self._unprocessed:
+            # Finally, handle anything that wasn't processed.
+            self.resend_unprocessed()
 
     def put_item(self, data, overwrite=False):
         self._to_put.append(data)
@@ -1122,7 +1125,43 @@ def flush(self):
                 }
             })
 
-        self.table.connection.batch_write_item(batch_data)
+        resp = self.table.connection.batch_write_item(batch_data)
+        self.handle_unprocessed(resp)
+
         self._to_put = []
         self._to_delete = []
         return True
+
+    def handle_unprocessed(self, resp):
+        if len(resp.get('UnprocessedItems', [])):
+            table_name = self.table.table_name
+            unprocessed = resp['UnprocessedItems'].get(table_name, [])
+
+            # Some items have not been processed. Stow them for now &
+            # re-attempt processing on ``__exit__``.
+            msg = "%s items were unprocessed. Storing for later."
+            boto.log.info(msg % len(unprocessed))
+            self._unprocessed.extend(unprocessed)
+
+    def resend_unprocessed(self):
+        # If there are unprocessed records (for instance, the user was over
+        # their throughput limitations), iterate over them & send until they're
+        # all there.
+        boto.log.info(
+            "Re-sending %s unprocessed items." % len(self._unprocessed)
+        )
+
+        while len(self._unprocessed):
+            # Again, do 25 at a time.
+            to_resend = self._unprocessed[:25]
+            # Remove them from the list.
+            self._unprocessed = self._unprocessed[25:]
+            batch_data = {
+                self.table.table_name: to_resend
+            }
+            boto.log.info("Sending %s items" % len(to_resend))
+            resp = self.table.connection.batch_write_item(batch_data)
+            self.handle_unprocessed(resp)
+            boto.log.info(
+                "%s unprocessed items left" % len(self._unprocessed)
+            )
\ No newline at end of file


Test patch:
diff --git a/tests/integration/dynamodb2/test_highlevel.py b/tests/integration/dynamodb2/test_highlevel.py
--- a/tests/integration/dynamodb2/test_highlevel.py
+++ b/tests/integration/dynamodb2/test_highlevel.py
@@ -314,3 +314,30 @@ def test_integration(self):
             'friends': set([]),
         })
         self.assertTrue(penny_created)
+
+    def test_unprocessed_batch_writes(self):
+        # Create a very limited table w/ low throughput.
+        users = Table.create('slow_users', schema=[
+            HashKey('user_id'),
+        ], throughput={
+            'read': 1,
+            'write': 1,
+        })
+        self.addCleanup(users.delete)
+
+        # Wait for it.
+        time.sleep(60)
+
+        with users.batch_write() as batch:
+            for i in range(500):
+                batch.put_item(data={
+                    'user_id': str(i),
+                    'name': 'Droid #{0}'.format(i),
+                })
+
+            # Before ``__exit__`` runs, we should have a bunch of unprocessed
+            # items.
+            self.assertTrue(len(batch._unprocessed) > 0)
+
+        # Post-__exit__, they should all be gone.
+        self.assertEqual(len(batch._unprocessed), 0)
diff --git a/tests/unit/dynamodb2/test_table.py b/tests/unit/dynamodb2/test_table.py
--- a/tests/unit/dynamodb2/test_table.py
+++ b/tests/unit/dynamodb2/test_table.py
@@ -1476,6 +1476,89 @@ def test_batch_write_flushing(self):
 
         self.assertEqual(mock_batch.call_count, 2)
 
+    def test_batch_write_unprocessed_items(self):
+        unprocessed = {
+            'UnprocessedItems': {
+                'users': [
+                    {
+                        'PutRequest': {
+                            'username': {
+                                'S': 'jane',
+                            },
+                            'date_joined': {
+                                'N': 12342547
+                            }
+                        },
+                    },
+                ],
+            },
+        }
+
+        # Test enqueuing the unprocessed bits.
+        with mock.patch.object(
+                self.users.connection,
+                'batch_write_item',
+                return_value=unprocessed) as mock_batch:
+            with self.users.batch_write() as batch:
+                self.assertEqual(len(batch._unprocessed), 0)
+
+                # Trash the ``resend_unprocessed`` method so that we don't
+                # infinite loop forever here.
+                batch.resend_unprocessed = lambda: True
+
+                batch.put_item(data={
+                    'username': 'jane',
+                    'date_joined': 12342547
+                })
+                batch.delete_item(username='johndoe')
+                batch.put_item(data={
+                    'username': 'alice',
+                    'date_joined': 12342888
+                })
+
+            self.assertEqual(len(batch._unprocessed), 1)
+
+        # Now test resending those unprocessed items.
+        with mock.patch.object(
+                self.users.connection,
+                'batch_write_item',
+                return_value={}) as mock_batch:
+            with self.users.batch_write() as batch:
+                self.assertEqual(len(batch._unprocessed), 0)
+
+                # Toss in faked unprocessed items, as though a previous batch
+                # had failed.
+                batch._unprocessed = [
+                    {
+                        'PutRequest': {
+                            'username': {
+                                'S': 'jane',
+                            },
+                            'date_joined': {
+                                'N': 12342547
+                            }
+                        },
+                    },
+                ]
+
+                batch.put_item(data={
+                    'username': 'jane',
+                    'date_joined': 12342547
+                })
+                batch.delete_item(username='johndoe')
+                batch.put_item(data={
+                    'username': 'alice',
+                    'date_joined': 12342888
+                })
+
+                # Flush, to make sure everything has been processed.
+                # Unprocessed items should still be hanging around.
+                batch.flush()
+                self.assertEqual(len(batch._unprocessed), 1)
+
+            # Post-exit, this should be emptied.
+            self.assertEqual(len(batch._unprocessed), 0)
+
     def test__build_filters(self):
         filters = self.users._build_filters({
             'username__eq': 'johndoe',