Repository: googleapis/google-cloud-python

Issue:
Transactions are commited in CommitRequest.NON_TRANSACTIONAL mode
We have just been bitten by this using the latest `0.5.0`. 

We had the following piece of code, run in two different threads:

``` python
with dataset.transaction as t:
  entity = dataset.get(entity_key)
  archived_entity = _move_to_archive_key(entity)
  t.delete(entity.key)
  t.put(archived_entity)
  return archived_entity
```

We've seen two copies of the `archived_entity` for the same `entity`.  We've managed to reproduce it with two threads performing this concurrently (running separate Connections each, because httplib2 is not thread safe).

Having used Datastore's underlying technology before, I expected to get a `ConcurrentModificationException` on delete, and one of the threads throwing it, similar to what the Java API for AppEngine describes:
https://cloud.google.com/appengine/docs/java/datastore/transactions#Java_Uses_for_transactions

However, both of them succeed creating two copies of `archived_entity`, and there's no `ConcurrentModificationException` anywhere in `gcloud-python` code.

We've tracked down to the following:
- `Transaction#commit` calls the `Batch#commit` https://github.com/GoogleCloudPlatform/gcloud-python/blob/master/gcloud/datastore/transaction.py#L177
- `Batch#commit` is unaware of the `Transaction#id`, calls `Connection#commit` without the `transaction_id` parameter https://github.com/GoogleCloudPlatform/gcloud-python/blob/master/gcloud/datastore/batch.py#L214
- `Connection#commit`gets `transaction_id`, and sets the REST request's mode to `datastore_pb.CommitRequest.NON_TRANSACTIONAL` in https://github.com/GoogleCloudPlatform/gcloud-python/blob/master/gcloud/datastore/connection.py#L321

_Impact_: This means that **all** transactions made through `python-gcloud` are **non-transactional**. Please treat it as a P0 bug.



Discussion:
We managed to get the correct behaviour by plumbing through `Transaction#id` through `Batch#commit`. The result is that the above test code works as intended: one thread doest the complete transaction, the other one gets:

``` python
 File "~/src/python/improbable/db/common/transaction.py", line 15, in commit
   response = self.connection.commit(self._dataset_id, self.mutation, self._id)
 File "~/virtualenv/python/local/lib/python2.7/site-packages/gcloud/datastore/connection.py", line 324, in commit
   datastore_pb.CommitResponse)
 File "~/virtualenv/python/local/lib/python2.7/site-packages/gcloud/datastore/connection.py", line 108, in _rpc
   data=request_pb.SerializeToString())
 File "~/virtualenv/python/local/lib/python2.7/site-packages/gcloud/datastore/connection.py", line 85, in _request
   raise make_exception(headers, content, use_json=False)
Conflict: 409 too much contention on these datastore entities. please try again.
```

I think the right solution here would be to catch it in the `Transaction#commit` and rethrow it as `ConcurrentModificationException`.


Resolving patch:
diff --git a/gcloud/datastore/batch.py b/gcloud/datastore/batch.py
--- a/gcloud/datastore/batch.py
+++ b/gcloud/datastore/batch.py
@@ -62,6 +62,7 @@ class Batch(object):
       ...   do_some_work(batch)
       ...   raise Exception() # rolls back
     """
+    _id = None  # "protected" attribute, always None for non-transactions
 
     def __init__(self, dataset_id=None, connection=None):
         """Construct a batch.
@@ -211,7 +212,8 @@ def commit(self):
         however it can be called explicitly if you don't want to use a
         context manager.
         """
-        response = self.connection.commit(self._dataset_id, self.mutation)
+        response = self.connection.commit(
+            self._dataset_id, self.mutation, self._id)
         # If the back-end returns without error, we are guaranteed that
         # the response's 'insert_auto_id_key' will match (length and order)
         # the request's 'insert_auto_id` entities, which are derived from
diff --git a/gcloud/datastore/connection.py b/gcloud/datastore/connection.py
--- a/gcloud/datastore/connection.py
+++ b/gcloud/datastore/connection.py
@@ -297,7 +297,7 @@ def begin_transaction(self, dataset_id, serializable=False):
 
         return response.transaction
 
-    def commit(self, dataset_id, mutation_pb, transaction_id=None):
+    def commit(self, dataset_id, mutation_pb, transaction_id):
         """Commit dataset mutations in context of current transation (if any).
 
         Maps the ``DatastoreService.Commit`` protobuf RPC.
@@ -308,10 +308,10 @@ def commit(self, dataset_id, mutation_pb, transaction_id=None):
         :type mutation_pb: :class:`datastore_pb.Mutation`.
         :param mutation_pb: The protobuf for the mutations being saved.
 
-        :type transaction_id: string
+        :type transaction_id: string or None
         :param transaction_id: The transaction ID returned from
-                               :meth:`begin_transaction`.  If not passed, the
-                               commit will be non-transactional.
+                               :meth:`begin_transaction`.  Non-transactional
+                               batches must pass ``None``.
 
         :rtype: :class:`gcloud.datastore._datastore_v1_pb2.MutationResult`.
         :returns': the result protobuf for the mutation.


Test patch:
diff --git a/gcloud/datastore/test_api.py b/gcloud/datastore/test_api.py
--- a/gcloud/datastore/test_api.py
+++ b/gcloud/datastore/test_api.py
@@ -663,7 +663,7 @@ def test_no_dataset_id(self):
 
         self.assertEqual(result, None)
         self.assertEqual(len(connection._committed), 1)
-        dataset_id, mutation = connection._committed[0]
+        dataset_id, mutation, transaction_id = connection._committed[0]
         self.assertEqual(dataset_id, _DATASET)
         upserts = list(mutation.upsert)
         self.assertEqual(len(upserts), 1)
@@ -671,6 +671,7 @@ def test_no_dataset_id(self):
         properties = list(upserts[0].property)
         self.assertEqual(properties[0].name, 'foo')
         self.assertEqual(properties[0].value.string_value, u'bar')
+        self.assertTrue(transaction_id is None)
 
     def test_no_entities(self):
         from gcloud.datastore import _implicit_environ
@@ -700,7 +701,7 @@ def test_no_batch_w_partial_key(self):
                                dataset_id=_DATASET)
         self.assertEqual(result, None)
         self.assertEqual(len(connection._committed), 1)
-        dataset_id, mutation = connection._committed[0]
+        dataset_id, mutation, transaction_id = connection._committed[0]
         self.assertEqual(dataset_id, _DATASET)
         inserts = list(mutation.insert_auto_id)
         self.assertEqual(len(inserts), 1)
@@ -708,6 +709,7 @@ def test_no_batch_w_partial_key(self):
         properties = list(inserts[0].property)
         self.assertEqual(properties[0].name, 'foo')
         self.assertEqual(properties[0].value.string_value, u'bar')
+        self.assertTrue(transaction_id is None)
 
     def test_existing_batch_w_completed_key(self):
         from gcloud.datastore.test_batch import _Connection
@@ -848,9 +850,10 @@ def test_no_dataset_id(self):
 
         self.assertEqual(result, None)
         self.assertEqual(len(connection._committed), 1)
-        dataset_id, mutation = connection._committed[0]
+        dataset_id, mutation, transaction_id = connection._committed[0]
         self.assertEqual(dataset_id, _DATASET)
         self.assertEqual(list(mutation.delete), [key.to_protobuf()])
+        self.assertTrue(transaction_id is None)
 
     def test_no_keys(self):
         from gcloud.datastore import _implicit_environ
@@ -872,9 +875,10 @@ def test_no_batch(self):
                                dataset_id=_DATASET)
         self.assertEqual(result, None)
         self.assertEqual(len(connection._committed), 1)
-        dataset_id, mutation = connection._committed[0]
+        dataset_id, mutation, transaction_id = connection._committed[0]
         self.assertEqual(dataset_id, _DATASET)
         self.assertEqual(list(mutation.delete), [key.to_protobuf()])
+        self.assertTrue(transaction_id is None)
 
     def test_wo_batch_w_key_different_than_default_dataset_id(self):
         from gcloud.datastore._testing import _monkey_defaults
@@ -892,9 +896,10 @@ def test_wo_batch_w_key_different_than_default_dataset_id(self):
             result = self._callFUT([key])
         self.assertEqual(result, None)
         self.assertEqual(len(connection._committed), 1)
-        dataset_id, mutation = connection._committed[0]
+        dataset_id, mutation, transaction_id = connection._committed[0]
         self.assertEqual(dataset_id, _DATASET)
         self.assertEqual(list(mutation.delete), [key.to_protobuf()])
+        self.assertTrue(transaction_id is None)
 
     def test_w_existing_batch(self):
         from gcloud.datastore.test_batch import _Connection
@@ -989,9 +994,10 @@ def test_no_batch(self):
                                dataset_id=_DATASET)
         self.assertEqual(result, None)
         self.assertEqual(len(connection._committed), 1)
-        dataset_id, mutation = connection._committed[0]
+        dataset_id, mutation, transaction_id = connection._committed[0]
         self.assertEqual(dataset_id, _DATASET)
         self.assertEqual(list(mutation.delete), [key.to_protobuf()])
+        self.assertTrue(transaction_id is None)
 
 
 class Test_allocate_ids_function(unittest2.TestCase):
diff --git a/gcloud/datastore/test_batch.py b/gcloud/datastore/test_batch.py
--- a/gcloud/datastore/test_batch.py
+++ b/gcloud/datastore/test_batch.py
@@ -42,6 +42,7 @@ def test_ctor_explicit(self):
 
         self.assertEqual(batch.dataset_id, _DATASET)
         self.assertEqual(batch.connection, connection)
+        self.assertTrue(batch._id is None)
         self.assertTrue(isinstance(batch.mutation, Mutation))
         self.assertEqual(batch._auto_id_entities, [])
 
@@ -56,6 +57,7 @@ def test_ctor_implicit(self):
 
         self.assertEqual(batch.dataset_id, _DATASET)
         self.assertEqual(batch.connection, CONNECTION)
+        self.assertTrue(batch._id is None)
         self.assertTrue(isinstance(batch.mutation, Mutation))
         self.assertEqual(batch._auto_id_entities, [])
 
@@ -260,7 +262,8 @@ def test_commit(self):
 
         batch.commit()
 
-        self.assertEqual(connection._committed, [(_DATASET, batch.mutation)])
+        self.assertEqual(connection._committed,
+                         [(_DATASET, batch.mutation, None)])
 
     def test_commit_w_auto_id_entities(self):
         _DATASET = 'DATASET'
@@ -274,7 +277,8 @@ def test_commit_w_auto_id_entities(self):
 
         batch.commit()
 
-        self.assertEqual(connection._committed, [(_DATASET, batch.mutation)])
+        self.assertEqual(connection._committed,
+                         [(_DATASET, batch.mutation, None)])
         self.assertFalse(key.is_partial)
         self.assertEqual(key._id, _NEW_ID)
 
@@ -302,7 +306,8 @@ def test_as_context_mgr_wo_error(self):
         self.assertEqual(upserts[0].key, key._key)
         deletes = list(batch.mutation.delete)
         self.assertEqual(len(deletes), 0)
-        self.assertEqual(connection._committed, [(_DATASET, batch.mutation)])
+        self.assertEqual(connection._committed,
+                         [(_DATASET, batch.mutation, None)])
 
     def test_as_context_mgr_nested(self):
         from gcloud.datastore.batch import _BATCHES
@@ -346,8 +351,8 @@ def test_as_context_mgr_nested(self):
         self.assertEqual(len(deletes), 0)
 
         self.assertEqual(connection._committed,
-                         [(_DATASET, batch2.mutation),
-                          (_DATASET, batch1.mutation)])
+                         [(_DATASET, batch2.mutation, None),
+                          (_DATASET, batch1.mutation, None)])
 
     def test_as_context_mgr_w_error(self):
         from gcloud.datastore.batch import _BATCHES
@@ -406,8 +411,8 @@ def __init__(self, *new_keys):
         self._commit_result = _CommitResult(*new_keys)
         self._committed = []
 
-    def commit(self, dataset_id, mutation):
-        self._committed.append((dataset_id, mutation))
+    def commit(self, dataset_id, mutation, transaction_id):
+        self._committed.append((dataset_id, mutation, transaction_id))
         return self._commit_result
 
 
diff --git a/gcloud/datastore/test_connection.py b/gcloud/datastore/test_connection.py
--- a/gcloud/datastore/test_connection.py
+++ b/gcloud/datastore/test_connection.py
@@ -707,7 +707,7 @@ def test_commit_wo_transaction(self):
             'commit',
         ])
         http = conn._http = Http({'status': '200'}, rsp_pb.SerializeToString())
-        result = conn.commit(DATASET_ID, mutation)
+        result = conn.commit(DATASET_ID, mutation, None)
         self.assertEqual(result.index_updates, 0)
         self.assertEqual(list(result.insert_auto_id_key), [])
         cw = http._called_with
diff --git a/gcloud/datastore/test_transaction.py b/gcloud/datastore/test_transaction.py
--- a/gcloud/datastore/test_transaction.py
+++ b/gcloud/datastore/test_transaction.py
@@ -134,7 +134,7 @@ def test_commit_no_auto_ids(self):
         xact._mutation = mutation = object()
         xact.begin()
         xact.commit()
-        self.assertEqual(connection._committed, (_DATASET, mutation))
+        self.assertEqual(connection._committed, (_DATASET, mutation, 234))
         self.assertEqual(xact.id, None)
 
     def test_commit_w_auto_ids(self):
@@ -150,7 +150,7 @@ def test_commit_w_auto_ids(self):
         xact._mutation = mutation = object()
         xact.begin()
         xact.commit()
-        self.assertEqual(connection._committed, (_DATASET, mutation))
+        self.assertEqual(connection._committed, (_DATASET, mutation, 234))
         self.assertEqual(xact.id, None)
         self.assertEqual(entity.key.path, [{'kind': _KIND, 'id': _ID}])
 
@@ -162,7 +162,7 @@ def test_context_manager_no_raise(self):
         with xact:
             self.assertEqual(xact.id, 234)
             self.assertEqual(connection._begun, _DATASET)
-        self.assertEqual(connection._committed, (_DATASET, mutation))
+        self.assertEqual(connection._committed, (_DATASET, mutation, 234))
         self.assertEqual(xact.id, None)
 
     def test_context_manager_w_raise(self):
@@ -210,8 +210,8 @@ def begin_transaction(self, dataset_id):
     def rollback(self, dataset_id, transaction_id):
         self._rolled_back = dataset_id, transaction_id
 
-    def commit(self, dataset_id, mutation):
-        self._committed = (dataset_id, mutation)
+    def commit(self, dataset_id, mutation, transaction_id):
+        self._committed = (dataset_id, mutation, transaction_id)
         return self._commit_result