Repository: googleapis/google-cloud-python

Issue:
spanner failed transaction being reused in next call to database.run_in_transaction()
Using google-cloud-spanner 0.28.0, when running the following:

```python
"""
DDL:

CREATE TABLE test (
	test_id STRING(MAX),
) PRIMARY KEY (test_id)
"""

instance = client.instance(INSTANCE_ID)
pool = spanner.BurstyPool(target_size=10)
database = instance.database(DATABASE_ID, pool=pool)

def unit_of_work(transaction, id):
    transaction.insert(
        table='test',
        columns=['test_id'],
        values=[
            [id],
        ])

print('#### inserting id_1')
database.run_in_transaction(unit_of_work, 'id_1')
print()

print('#### inserting id_1 again, expected to fail')
try:
    database.run_in_transaction(unit_of_work, 'id_1')
except Exception as exc:
    print('#### failed as expected: {}'.format(exc))
else:
    raise Exception
print()

print('#### inserting id_2')
database.run_in_transaction(unit_of_work, 'id_2')
```

The last line fails and appears to be retrying the previous failed transaction.

Below is the output, note the `StatusCode.ALREADY_EXISTS, Row [id_1] in table test.` referencing id_1 when it fails which was from the previous failed transaction, not the one currently being executed.

```
#### inserting id_1

#### inserting id_1 again, expected to fail
#### failed as expected: RetryError(Exception occurred in retry method that was not classified as transient, caused by <_Rendezvous of RPC that terminated with (StatusCode.ALREADY_EXISTS, Row [id_1] in table test.)>)

#### inserting id_2
Traceback (most recent call last):
  File "/opt/sp/python3.6/lib/python3.6/site-packages/google/gax/retry.py", line 121, in inner
    return to_call(*args)
  File "/opt/sp/python3.6/lib/python3.6/site-packages/google/gax/retry.py", line 68, in inner
    return a_func(*updated_args, **kwargs)
  File "/opt/sp/python3.6/lib/python3.6/site-packages/grpc/_channel.py", line 492, in __call__
    return _end_unary_response_blocking(state, call, False, deadline)
  File "/opt/sp/python3.6/lib/python3.6/site-packages/grpc/_channel.py", line 440, in _end_unary_response_blocking
    raise _Rendezvous(state, None, None, deadline)
grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with (StatusCode.ALREADY_EXISTS, Row [id_1] in table test.)>

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "./dev/transaction_bug.py", line 40, in <module>
    database.run_in_transaction(unit_of_work, 'id_2')
  File "/opt/sp/python3.6/lib/python3.6/site-packages/google/cloud/spanner_v1/database.py", line 378, in run_in_transaction
    return session.run_in_transaction(func, *args, **kw)
  File "/opt/sp/python3.6/lib/python3.6/site-packages/google/cloud/spanner_v1/session.py", line 293, in run_in_transaction
    _delay_until_retry(exc, deadline)
  File "/opt/sp/python3.6/lib/python3.6/site-packages/google/cloud/spanner_v1/session.py", line 291, in run_in_transaction
    txn.commit()
  File "/opt/sp/python3.6/lib/python3.6/site-packages/google/cloud/spanner_v1/transaction.py", line 124, in commit
    transaction_id=self._transaction_id, options=options)
  File "/opt/sp/python3.6/lib/python3.6/site-packages/google/cloud/spanner_v1/gapic/spanner_client.py", line 845, in commit
    return self._commit(request, options)
  File "/opt/sp/python3.6/lib/python3.6/site-packages/google/gax/api_callable.py", line 452, in inner
    return api_caller(api_call, this_settings, request)
  File "/opt/sp/python3.6/lib/python3.6/site-packages/google/gax/api_callable.py", line 438, in base_caller
    return api_call(*args)
  File "/opt/sp/python3.6/lib/python3.6/site-packages/google/gax/api_callable.py", line 376, in inner
    return a_func(*args, **kwargs)
  File "/opt/sp/python3.6/lib/python3.6/site-packages/google/gax/retry.py", line 127, in inner
    ' classified as transient', exception)
google.gax.errors.RetryError: RetryError(Exception occurred in retry method that was not classified as transient, caused by <_Rendezvous of RPC that terminated with (StatusCode.ALREADY_EXISTS, Row [id_1] in table test.)>)
```

If this is intended behavior, how should the client code clear the failed transaction so future calls to database.run_in_transaction() don't keep failing?

This seems similar to #3434. 


Discussion:
@lukesneeringer I believe this needs to be P0 . This makes the client unusable if the issue is as described.
I'm working on a system test to replicate the reported issue.
I am able to reproduce this and this only happens if the previous attempt failed. I think this is because the client deletes the transaction object after calling `_delay_until_retry`. But `_delay_until_retry` will `raise` if the error is not `ABORTED` so the transaction object will never get deleted.
https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/spanner/google/cloud/spanner_v1/session.py#L294

Resolving patch:
diff --git a/spanner/google/cloud/spanner_v1/session.py b/spanner/google/cloud/spanner_v1/session.py
--- a/spanner/google/cloud/spanner_v1/session.py
+++ b/spanner/google/cloud/spanner_v1/session.py
@@ -280,8 +280,8 @@ def run_in_transaction(self, func, *args, **kw):
             try:
                 return_value = func(txn, *args, **kw)
             except (GaxError, GrpcRendezvous) as exc:
-                _delay_until_retry(exc, deadline)
                 del self._transaction
+                _delay_until_retry(exc, deadline)
                 continue
             except Exception:
                 txn.rollback()
@@ -290,8 +290,8 @@ def run_in_transaction(self, func, *args, **kw):
             try:
                 txn.commit()
             except GaxError as exc:
-                _delay_until_retry(exc, deadline)
                 del self._transaction
+                _delay_until_retry(exc, deadline)
             else:
                 return return_value
 
diff --git a/spanner/nox.py b/spanner/nox.py
--- a/spanner/nox.py
+++ b/spanner/nox.py
@@ -72,10 +72,10 @@ def system_tests(session, python_version):
     # virtualenv's dist-packages.
     session.install('mock', 'pytest', *LOCAL_DEPS)
     session.install('../test_utils/')
-    session.install('.')
+    session.install('-e', '.')
 
     # Run py.test against the system tests.
-    session.run('py.test', '--quiet', 'tests/system')
+    session.run('py.test', '--quiet', 'tests/system', *session.posargs)
 
 
 @nox.session


Test patch:
diff --git a/spanner/tests/system/test_system.py b/spanner/tests/system/test_system.py
--- a/spanner/tests/system/test_system.py
+++ b/spanner/tests/system/test_system.py
@@ -30,6 +30,9 @@
 from google.cloud.spanner_v1.proto.type_pb2 import STRING
 from google.cloud.spanner_v1.proto.type_pb2 import TIMESTAMP
 from google.cloud.spanner_v1.proto.type_pb2 import Type
+from google.gax.grpc import exc_to_code
+from google.gax import errors
+from grpc import StatusCode
 
 from google.cloud._helpers import UTC
 from google.cloud.exceptions import GrpcRendezvous
@@ -73,7 +76,6 @@ class Config(object):
 
 def _retry_on_unavailable(exc):
     """Retry only errors whose status code is 'UNAVAILABLE'."""
-    from grpc import StatusCode
     return exc.code() == StatusCode.UNAVAILABLE
 
 
@@ -347,6 +349,31 @@ def _unit_of_work(transaction, test):
             rows = list(after.execute_sql(self.SQL))
         self._check_row_data(rows)
 
+    def test_db_run_in_transaction_twice_4181(self):
+        retry = RetryInstanceState(_has_all_ddl)
+        retry(self._db.reload)()
+
+        with self._db.batch() as batch:
+            batch.delete(COUNTERS_TABLE, self.ALL)
+
+        def _unit_of_work(transaction, name):
+            transaction.insert(COUNTERS_TABLE, COUNTERS_COLUMNS, [[name, 0]])
+
+        self._db.run_in_transaction(_unit_of_work, name='id_1')
+
+        with self.assertRaises(errors.RetryError) as expected:
+            self._db.run_in_transaction(_unit_of_work, name='id_1')
+
+        self.assertEqual(
+            exc_to_code(expected.exception.cause), StatusCode.ALREADY_EXISTS)
+
+        self._db.run_in_transaction(_unit_of_work, name='id_2')
+
+        with self._db.snapshot() as after:
+            rows = list(after.read(
+                COUNTERS_TABLE, COUNTERS_COLUMNS, self.ALL))
+        self.assertEqual(len(rows), 2)
+
 
 class TestSessionAPI(unittest.TestCase, _TestData):
     ALL_TYPES_TABLE = 'all_types'
diff --git a/spanner/tests/unit/test_session.py b/spanner/tests/unit/test_session.py
--- a/spanner/tests/unit/test_session.py
+++ b/spanner/tests/unit/test_session.py
@@ -395,8 +395,9 @@ def test_transaction_w_existing_txn(self):
         self.assertIs(session._transaction, another)
         self.assertTrue(existing._rolled_back)
 
-    def test_retry_transaction_w_commit_error_txn_already_begun(self):
-        from google.gax.errors import GaxError
+    def test_run_in_transaction_callback_raises_non_gax_error(self):
+        from google.cloud.spanner_v1.proto.transaction_pb2 import (
+            Transaction as TransactionPB)
         from google.cloud.spanner_v1.transaction import Transaction
 
         TABLE_NAME = 'citizens'
@@ -405,33 +406,41 @@ def test_retry_transaction_w_commit_error_txn_already_begun(self):
             ['phred@exammple.com', 'Phred', 'Phlyntstone', 32],
             ['bharney@example.com', 'Bharney', 'Rhubble', 31],
         ]
+        TRANSACTION_ID = b'FACEDACE'
+        transaction_pb = TransactionPB(id=TRANSACTION_ID)
         gax_api = _SpannerApi(
-            _commit_error=True,
+            _begin_transaction_response=transaction_pb,
+            _rollback_response=None,
         )
         database = _Database(self.DATABASE_NAME)
         database.spanner_api = gax_api
         session = self._make_one(database)
         session._session_id = 'DEADBEEF'
-        begun_txn = session._transaction = Transaction(session)
-        begun_txn._transaction_id = b'FACEDACE'
 
         called_with = []
 
+        class Testing(Exception):
+            pass
+
         def unit_of_work(txn, *args, **kw):
             called_with.append((txn, args, kw))
             txn.insert(TABLE_NAME, COLUMNS, VALUES)
+            raise Testing()
 
-        with self.assertRaises(GaxError):
+        with self.assertRaises(Testing):
             session.run_in_transaction(unit_of_work)
 
+        self.assertIsNone(session._transaction)
         self.assertEqual(len(called_with), 1)
         txn, args, kw = called_with[0]
-        self.assertIs(txn, begun_txn)
-        self.assertEqual(txn.committed, None)
+        self.assertIsInstance(txn, Transaction)
+        self.assertIsNone(txn.committed)
+        self.assertTrue(txn._rolled_back)
         self.assertEqual(args, ())
         self.assertEqual(kw, {})
 
-    def test_run_in_transaction_callback_raises_abort(self):
+    def test_run_in_transaction_callback_raises_gax_error_non_abort(self):
+        from google.gax.errors import GaxError
         from google.cloud.spanner_v1.proto.transaction_pb2 import (
             Transaction as TransactionPB)
         from google.cloud.spanner_v1.transaction import Transaction
@@ -455,22 +464,23 @@ def test_run_in_transaction_callback_raises_abort(self):
 
         called_with = []
 
-        class Testing(Exception):
+        class Testing(GaxError):
             pass
 
         def unit_of_work(txn, *args, **kw):
             called_with.append((txn, args, kw))
             txn.insert(TABLE_NAME, COLUMNS, VALUES)
-            raise Testing()
+            raise Testing('testing')
 
         with self.assertRaises(Testing):
             session.run_in_transaction(unit_of_work)
 
+        self.assertIsNone(session._transaction)
         self.assertEqual(len(called_with), 1)
         txn, args, kw = called_with[0]
         self.assertIsInstance(txn, Transaction)
         self.assertIsNone(txn.committed)
-        self.assertTrue(txn._rolled_back)
+        self.assertFalse(txn._rolled_back)
         self.assertEqual(args, ())
         self.assertEqual(kw, {})
 
@@ -521,6 +531,43 @@ def unit_of_work(txn, *args, **kw):
         self.assertEqual(args, ('abc',))
         self.assertEqual(kw, {'some_arg': 'def'})
 
+    def test_run_in_transaction_w_commit_error(self):
+        from google.gax.errors import GaxError
+        from google.cloud.spanner_v1.transaction import Transaction
+
+        TABLE_NAME = 'citizens'
+        COLUMNS = ['email', 'first_name', 'last_name', 'age']
+        VALUES = [
+            ['phred@exammple.com', 'Phred', 'Phlyntstone', 32],
+            ['bharney@example.com', 'Bharney', 'Rhubble', 31],
+        ]
+        gax_api = _SpannerApi(
+            _commit_error=True,
+        )
+        database = _Database(self.DATABASE_NAME)
+        database.spanner_api = gax_api
+        session = self._make_one(database)
+        session._session_id = 'DEADBEEF'
+        begun_txn = session._transaction = Transaction(session)
+        begun_txn._transaction_id = b'FACEDACE'
+
+        called_with = []
+
+        def unit_of_work(txn, *args, **kw):
+            called_with.append((txn, args, kw))
+            txn.insert(TABLE_NAME, COLUMNS, VALUES)
+
+        with self.assertRaises(GaxError):
+            session.run_in_transaction(unit_of_work)
+
+        self.assertIsNone(session._transaction)
+        self.assertEqual(len(called_with), 1)
+        txn, args, kw = called_with[0]
+        self.assertIs(txn, begun_txn)
+        self.assertEqual(txn.committed, None)
+        self.assertEqual(args, ())
+        self.assertEqual(kw, {})
+
     def test_run_in_transaction_w_abort_no_retry_metadata(self):
         import datetime
         from google.cloud.spanner_v1.proto.spanner_pb2 import CommitResponse