Repository: googleapis/python-spanner

Issue:
spanner: InvalidArgument: 400 Previously received a different request with this seqno. seqno=4 with no concurrency applied
With spanner_v1 version 1.11, I am using this package without any concurrency and yet I am getting back an obscure error that looks to me like an issue with the underlying gRPC library or the coordination in this library
```shell
google.api_core.exceptions.InvalidArgument: 400 Previously received a different request with this seqno. seqno=4
```
and that finally results in https://github.com/googleapis/python-spanner/issues/10 even after I've removed PingingPool to use the default pool.

Please find the full stack trace below in the details element
<details>

```shell
======================================================================
ERROR: test_xview_class (admin_docs.test_middleware.XViewMiddlewareTest)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 57, in error_remapped_callable
    return callable_(*args, **kwargs)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/grpc/_channel.py", line 824, in __call__
    return _end_unary_response_blocking(state, call, False, None)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/grpc/_channel.py", line 726, in _end_unary_response_blocking
    raise _InactiveRpcError(state)
grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
	status = StatusCode.INVALID_ARGUMENT
	details = "Previously received a different request with this seqno. seqno=4"
	debug_error_string = "{"created":"@1580855981.982223904","description":"Error received from peer ipv4:108.177.13.95:443","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Previously received a different request with this seqno. seqno=4","grpc_status":3}"
>
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 89, in execute
    self.__handle_update(self.__get_txn(), sql, args or None)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 101, in __handle_update
    res = txn.execute_update(sql, params=params, param_types=get_param_types(params))
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/cloud/spanner_v1/transaction.py", line 202, in execute_update
    metadata=metadata,
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/cloud/spanner_v1/gapic/spanner_client.py", line 810, in execute_sql
    request, retry=retry, timeout=timeout, metadata=metadata
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/gapic_v1/method.py", line 143, in __call__
    return wrapped_func(*args, **kwargs)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/retry.py", line 286, in retry_wrapped_func
    on_error=on_error,
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/retry.py", line 184, in retry_target
    return target()
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/timeout.py", line 214, in func_with_timeout
    return func(*args, **kwargs)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 59, in error_remapped_callable
    six.raise_from(exceptions.from_grpc_error(exc), exc)
  File "<string>", line 3, in raise_from
google.api_core.exceptions.InvalidArgument: 400 Previously received a different request with this seqno. seqno=4
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 93, in execute
    raise ProgrammingError(e.details if hasattr(e, 'details') else e)
spanner.dbapi.exceptions.ProgrammingError: 400 Previously received a different request with this seqno. seqno=4
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/test/testcases.py", line 267, in __call__
    self._pre_setup()
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/test/testcases.py", line 938, in _pre_setup
    self._fixture_setup()
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/test/testcases.py", line 1165, in _fixture_setup
    self.setUpTestData()
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/tests/admin_docs/tests.py", line 11, in setUpTestData
    cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/contrib/auth/models.py", line 162, in create_superuser
    return self._create_user(username, email, password, **extra_fields)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/contrib/auth/models.py", line 145, in _create_user
    user.save(using=self._db)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/contrib/auth/base_user.py", line 66, in save
    super().save(*args, **kwargs)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/base.py", line 741, in save
    force_update=force_update, update_fields=update_fields)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/base.py", line 779, in save_base
    force_update, using, update_fields,
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/base.py", line 851, in _save_table
    forced_update)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/base.py", line 900, in _do_update
    return filtered._update(values) > 0
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/query.py", line 760, in _update
    return query.get_compiler(self.db).execute_sql(CURSOR)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/sql/compiler.py", line 1462, in execute_sql
    cursor = super().execute_sql(result_type)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/sql/compiler.py", line 1133, in execute_sql
    cursor.execute(sql, params)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/backends/utils.py", line 67, in execute
    return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/backends/utils.py", line 76, in _execute_with_wrappers
    return executor(sql, params, many, context)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/utils.py", line 89, in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 93, in execute
    raise ProgrammingError(e.details if hasattr(e, 'details') else e)
django.db.utils.ProgrammingError: 400 Previously received a different request with this seqno. seqno=4
======================================================================
ERROR: test_xview_func (admin_docs.test_middleware.XViewMiddlewareTest)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 57, in error_remapped_callable
    return callable_(*args, **kwargs)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/grpc/_channel.py", line 824, in __call__
    return _end_unary_response_blocking(state, call, False, None)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/grpc/_channel.py", line 726, in _end_unary_response_blocking
    raise _InactiveRpcError(state)
grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
	status = StatusCode.INVALID_ARGUMENT
	details = "Previously received a different request with this seqno. seqno=4"
	debug_error_string = "{"created":"@1580855981.991133481","description":"Error received from peer ipv4:108.177.13.95:443","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Previously received a different request with this seqno. seqno=4","grpc_status":3}"
>
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 89, in execute
    self.__handle_update(self.__get_txn(), sql, args or None)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 101, in __handle_update
    res = txn.execute_update(sql, params=params, param_types=get_param_types(params))
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/cloud/spanner_v1/transaction.py", line 202, in execute_update
    metadata=metadata,
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/cloud/spanner_v1/gapic/spanner_client.py", line 810, in execute_sql
    request, retry=retry, timeout=timeout, metadata=metadata
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/gapic_v1/method.py", line 143, in __call__
    return wrapped_func(*args, **kwargs)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/retry.py", line 286, in retry_wrapped_func
    on_error=on_error,
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/retry.py", line 184, in retry_target
    return target()
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/timeout.py", line 214, in func_with_timeout
    return func(*args, **kwargs)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 59, in error_remapped_callable
    six.raise_from(exceptions.from_grpc_error(exc), exc)
  File "<string>", line 3, in raise_from
google.api_core.exceptions.InvalidArgument: 400 Previously received a different request with this seqno. seqno=4
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 93, in execute
    raise ProgrammingError(e.details if hasattr(e, 'details') else e)
spanner.dbapi.exceptions.ProgrammingError: 400 Previously received a different request with this seqno. seqno=4
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/test/testcases.py", line 267, in __call__
    self._pre_setup()
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/test/testcases.py", line 938, in _pre_setup
    self._fixture_setup()
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/test/testcases.py", line 1165, in _fixture_setup
    self.setUpTestData()
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/tests/admin_docs/tests.py", line 11, in setUpTestData
    cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/contrib/auth/models.py", line 162, in create_superuser
    return self._create_user(username, email, password, **extra_fields)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/contrib/auth/models.py", line 145, in _create_user
    user.save(using=self._db)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/contrib/auth/base_user.py", line 66, in save
    super().save(*args, **kwargs)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/base.py", line 741, in save
    force_update=force_update, update_fields=update_fields)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/base.py", line 779, in save_base
    force_update, using, update_fields,
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/base.py", line 851, in _save_table
    forced_update)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/base.py", line 900, in _do_update
    return filtered._update(values) > 0
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/query.py", line 760, in _update
    return query.get_compiler(self.db).execute_sql(CURSOR)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/sql/compiler.py", line 1462, in execute_sql
    cursor = super().execute_sql(result_type)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/sql/compiler.py", line 1133, in execute_sql
    cursor.execute(sql, params)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/backends/utils.py", line 67, in execute
    return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/backends/utils.py", line 76, in _execute_with_wrappers
    return executor(sql, params, many, context)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/utils.py", line 89, in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 93, in execute
    raise ProgrammingError(e.details if hasattr(e, 'details') else e)
django.db.utils.ProgrammingError: 400 Previously received a different request with this seqno. seqno=4
======================================================================
ERROR: test_bookmarklets (admin_docs.test_views.AdminDocViewTests)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 57, in error_remapped_callable
    return callable_(*args, **kwargs)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/grpc/_channel.py", line 824, in __call__
    return _end_unary_response_blocking(state, call, False, None)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/grpc/_channel.py", line 726, in _end_unary_response_blocking
    raise _InactiveRpcError(state)
grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
	status = StatusCode.INVALID_ARGUMENT
	details = "Previously received a different request with this seqno. seqno=4"
	debug_error_string = "{"created":"@1580855982.000827256","description":"Error received from peer ipv4:108.177.13.95:443","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Previously received a different request with this seqno. seqno=4","grpc_status":3}"
>
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 89, in execute
    self.__handle_update(self.__get_txn(), sql, args or None)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 101, in __handle_update
    res = txn.execute_update(sql, params=params, param_types=get_param_types(params))
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/cloud/spanner_v1/transaction.py", line 202, in execute_update
    metadata=metadata,
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/cloud/spanner_v1/gapic/spanner_client.py", line 810, in execute_sql
    request, retry=retry, timeout=timeout, metadata=metadata
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/gapic_v1/method.py", line 143, in __call__
    return wrapped_func(*args, **kwargs)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/retry.py", line 286, in retry_wrapped_func
    on_error=on_error,
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/retry.py", line 184, in retry_target
    return target()
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/timeout.py", line 214, in func_with_timeout
    return func(*args, **kwargs)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 59, in error_remapped_callable
    six.raise_from(exceptions.from_grpc_error(exc), exc)
  File "<string>", line 3, in raise_from
google.api_core.exceptions.InvalidArgument: 400 Previously received a different request with this seqno. seqno=4
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 93, in execute
    raise ProgrammingError(e.details if hasattr(e, 'details') else e)
spanner.dbapi.exceptions.ProgrammingError: 400 Previously received a different request with this seqno. seqno=4
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/test/testcases.py", line 267, in __call__
    self._pre_setup()
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/test/testcases.py", line 938, in _pre_setup
    self._fixture_setup()
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/test/testcases.py", line 1165, in _fixture_setup
    self.setUpTestData()
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/tests/admin_docs/tests.py", line 11, in setUpTestData
    cls.superuser = User.objects.create_superuser(username='super', password='secret', email='super@example.com')
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/contrib/auth/models.py", line 162, in create_superuser
    return self._create_user(username, email, password, **extra_fields)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/contrib/auth/models.py", line 145, in _create_user
    user.save(using=self._db)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/contrib/auth/base_user.py", line 66, in save
    super().save(*args, **kwargs)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/base.py", line 741, in save
    force_update=force_update, update_fields=update_fields)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/base.py", line 779, in save_base
    force_update, using, update_fields,
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/base.py", line 851, in _save_table
    forced_update)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/base.py", line 900, in _do_update
    return filtered._update(values) > 0
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/query.py", line 760, in _update
    return query.get_compiler(self.db).execute_sql(CURSOR)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/sql/compiler.py", line 1462, in execute_sql
    cursor = super().execute_sql(result_type)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/models/sql/compiler.py", line 1133, in execute_sql
    cursor.execute(sql, params)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/backends/utils.py", line 67, in execute
    return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/backends/utils.py", line 76, in _execute_with_wrappers
    return executor(sql, params, many, context)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/utils.py", line 89, in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/spanner/dbapi/cursor.py", line 93, in execute
    raise ProgrammingError(e.details if hasattr(e, 'details') else e)
django.db.utils.ProgrammingError: 400 Previously received a different request with this seqno. seqno=4
======================================================================
ERROR: test_index (admin_docs.test_views.AdminDocViewTests)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 57, in error_remapped_callable
    return callable_(*args, **kwargs)
 ...
    raise ProgrammingError(e.details if hasattr(e, 'details') else e)
django.db.utils.ProgrammingError: 400 Previously received a different request with this seqno. seqno=4
======================================================================
ERROR: test_missing_docutils (admin_docs.test_views.AdminDocViewTests)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 57, in error_remapped_callable
    return callable_(*args, **kwargs)
...
django.db.utils.ProgrammingError: 400 Previously received a different request with this seqno. seqno=4
...
django.db.utils.ProgrammingError: 400 Previously received a different request with this seqno. seqno=4
======================================================================
ERROR: test_model_with_many_to_one (admin_docs.test_views.TestModelDetailView)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 57, in error_remapped_callable
    return callable_(*args, **kwargs)
...
django.db.utils.ProgrammingError: 400 Previously received a different request with this seqno. seqno=4
======================================================================
ERROR: test_model_with_no_backward_relations_render_only_relevant_fields (admin_docs.test_views.TestModelDetailView)
----------------------------------------------------------------------
...<SAME CONTENT>
django.db.utils.ProgrammingError: 400 Previously received a different request with this seqno. seqno=4
----------------------------------------------------------------------
Ran 13 tests in 0.927s
FAILED (errors=45)
Testing against Django installed in '/home/travis/build/orijtech/spanner-orm/django_tests/django/django' with up to 2 processes
Importing application admin_default_site
Importing application admin_docs
Skipping setup of unused database(s): other.
Operations to perform:
  Synchronize unmigrated apps: admin_default_site, admin_docs, auth, contenttypes, messages, sessions, staticfiles
  Apply all migrations: admin, sites
Synchronizing apps without migrations:
  Creating tables...
    Creating table django_content_type
    Creating table auth_permission
    Creating table auth_group
    Creating table auth_user
    Creating table django_session
    Creating table admin_docs_company
    Creating table admin_docs_group
    Creating table admin_docs_family
    Creating table admin_docs_person
    Running deferred SQL...
Running migrations:
  Applying admin.0001_initial... OK
  Applying admin.0002_logentry_remove_auto_add... OK
  Applying admin.0003_logentry_add_action_flag_choices... OK
  Applying sites.0001_initial... OK
  Applying sites.0002_alter_domain_unique... OK
System check identified no issues (0 silenced).
Traceback (most recent call last):
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 57, in error_remapped_callable
    return callable_(*args, **kwargs)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/grpc/_channel.py", line 824, in __call__
    return _end_unary_response_blocking(state, call, False, None)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/grpc/_channel.py", line 726, in _end_unary_response_blocking
    raise _InactiveRpcError(state)
grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
	status = StatusCode.ABORTED
	details = "Transaction not found"
	debug_error_string = "{"created":"@1580855984.140864504","description":"Error received from peer ipv4:108.177.13.95:443","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Transaction not found","grpc_status":10}"
>
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
  File "runtests.py", line 507, in <module>
    options.exclude_tags,
  File "runtests.py", line 294, in django_tests
    extra_tests=extra_tests,
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/test/runner.py", line 639, in run_tests
    self.teardown_databases(old_config)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/test/runner.py", line 583, in teardown_databases
    keepdb=self.keepdb,
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/test/utils.py", line 299, in teardown_databases
    connection.creation.destroy_test_db(old_name, verbosity, keepdb)
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/backends/base/creation.py", line 241, in destroy_test_db
    self.connection.close()
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/backends/base/base.py", line 288, in close
    self._close()
  File "/home/travis/build/orijtech/spanner-orm/django_tests/django/django/db/backends/base/base.py", line 250, in _close
    return self.connection.close()
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/spanner/dbapi/connection.py", line 30, in close
    self.commit()
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/spanner/dbapi/connection.py", line 60, in commit
    res = self.__txn.commit()
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/cloud/spanner_v1/transaction.py", line 127, in commit
    metadata=metadata,
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/cloud/spanner_v1/gapic/spanner_client.py", line 1556, in commit
    request, retry=retry, timeout=timeout, metadata=metadata
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/gapic_v1/method.py", line 143, in __call__
    return wrapped_func(*args, **kwargs)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/retry.py", line 286, in retry_wrapped_func
    on_error=on_error,
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/retry.py", line 184, in retry_target
    return target()
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/timeout.py", line 214, in func_with_timeout
    return func(*args, **kwargs)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 59, in error_remapped_callable
    six.raise_from(exceptions.from_grpc_error(exc), exc)
  File "<string>", line 3, in raise_from
google.api_core.exceptions.Aborted: 409 Transaction not found
The command "bash django_test_suite.sh" exited with 1.
cache.2
store build cache
```
</details>

Discussion:
googlebot: We found a Contributor License Agreement for you (the sender of this pull request), but were unable to find agreements for all the commit author(s) or Co-authors.  If you authored these, maybe you used a different email address in the git commits than was used to sign the CLA ([login here](https://cla.developers.google.com/) to double check)?  If these were authored by someone else, then they will need to sign a CLA as well, and confirm that they're okay with these being contributed to Google.
In order to pass this check, please resolve this problem and then comment `@googlebot I fixed it.`. If the bot doesn't comment, it means it doesn't think anything has changed.

ℹ️ **Googlers: [Go here](https://goto.google.com/prinfo/https%3A%2F%2Fgithub.com%2Fgoogleapis%2Fpython-spanner%2Fpull%2F19) for more info**.

<!-- need_author_cla -->
tseaver: @larkee I don't know what the CLA bot is on about:  `tseaver@palladion.com` is my normal github email, and the one associated with my long-ago-signed CLA (I've been committing to these repos using it for 5+ years now!).
tseaver: Also, the PR still needs the added unit tests I outlined above.
googlebot: CLAs look good, thanks!

ℹ️ **Googlers: [Go here](https://goto.google.com/prinfo/https%3A%2F%2Fgithub.com%2Fgoogleapis%2Fpython-spanner%2Fpull%2F19) for more info**.

<!-- ok -->
larkee: @tseaver I've fixed the CLA problem by just redoing the commits. I added the test and noticed that the change to snapshot.py doesn't actually make any difference, possibly because the call is wrapped in functools.partial? Because of this, I've reverted it.
odeke-em: Kindly pinging you @tseaver @skuruppu to take a look here as we need this change. Thank you.

Patch:
diff --git a/google/cloud/spanner_v1/transaction.py b/google/cloud/spanner_v1/transaction.py
index 5a161fd..27c2602 100644
--- a/google/cloud/spanner_v1/transaction.py
+++ b/google/cloud/spanner_v1/transaction.py
@@ -201,6 +201,11 @@ class Transaction(_SnapshotBase, _BatchBase):
         transaction = self._make_txn_selector()
         api = database.spanner_api
 
+        seqno, self._execute_sql_count = (
+            self._execute_sql_count,
+            self._execute_sql_count + 1,
+        )
+
         # Query-level options have higher precedence than client-level and
         # environment-level options
         default_query_options = database._instance._client._query_options
@@ -214,11 +219,9 @@ class Transaction(_SnapshotBase, _BatchBase):
             param_types=param_types,
             query_mode=query_mode,
             query_options=query_options,
-            seqno=self._execute_sql_count,
+            seqno=seqno,
             metadata=metadata,
         )
-
-        self._execute_sql_count += 1
         return response.stats.row_count_exact
 
     def batch_update(self, statements):
@@ -259,15 +262,18 @@ class Transaction(_SnapshotBase, _BatchBase):
         transaction = self._make_txn_selector()
         api = database.spanner_api
 
+        seqno, self._execute_sql_count = (
+            self._execute_sql_count,
+            self._execute_sql_count + 1,
+        )
+
         response = api.execute_batch_dml(
             session=self._session.name,
             transaction=transaction,
             statements=parsed,
-            seqno=self._execute_sql_count,
+            seqno=seqno,
             metadata=metadata,
         )
-
-        self._execute_sql_count += 1
         row_counts = [
             result_set.stats.row_count_exact for result_set in response.result_sets
         ]


Test patch:
diff --git a/tests/unit/test_snapshot.py b/tests/unit/test_snapshot.py
index e29b19d..40ba1c6 100644
--- a/tests/unit/test_snapshot.py
+++ b/tests/unit/test_snapshot.py
@@ -311,6 +311,8 @@ class Test_SnapshotBase(unittest.TestCase):
         with self.assertRaises(RuntimeError):
             list(derived.execute_sql(SQL_QUERY))
 
+        self.assertEqual(derived._execute_sql_count, 1)
+
     def test_execute_sql_w_params_wo_param_types(self):
         database = _Database()
         session = _Session(database)
diff --git a/tests/unit/test_transaction.py b/tests/unit/test_transaction.py
index dcb6cb9..6ae24ae 100644
--- a/tests/unit/test_transaction.py
+++ b/tests/unit/test_transaction.py
@@ -413,6 +413,19 @@ class TestTransaction(unittest.TestCase):
     def test_execute_update_w_count(self):
         self._execute_update_helper(count=1)
 
+    def test_execute_update_error(self):
+        database = _Database()
+        database.spanner_api = self._make_spanner_api()
+        database.spanner_api.execute_sql.side_effect = RuntimeError()
+        session = _Session(database)
+        transaction = self._make_one(session)
+        transaction._transaction_id = self.TRANSACTION_ID
+
+        with self.assertRaises(RuntimeError):
+            transaction.execute_update(DML_QUERY)
+
+        self.assertEqual(transaction._execute_sql_count, 1)
+
     def test_execute_update_w_query_options(self):
         from google.cloud.spanner_v1.proto.spanner_pb2 import ExecuteSqlRequest
 
@@ -513,6 +526,31 @@ class TestTransaction(unittest.TestCase):
     def test_batch_update_w_errors(self):
         self._batch_update_helper(error_after=2, count=1)
 
+    def test_batch_update_error(self):
+        database = _Database()
+        api = database.spanner_api = self._make_spanner_api()
+        api.execute_batch_dml.side_effect = RuntimeError()
+        session = _Session(database)
+        transaction = self._make_one(session)
+        transaction._transaction_id = self.TRANSACTION_ID
+
+        insert_dml = "INSERT INTO table(pkey, desc) VALUES (%pkey, %desc)"
+        insert_params = {"pkey": 12345, "desc": "DESCRIPTION"}
+        insert_param_types = {"pkey": "INT64", "desc": "STRING"}
+        update_dml = 'UPDATE table SET desc = desc + "-amended"'
+        delete_dml = "DELETE FROM table WHERE desc IS NULL"
+
+        dml_statements = [
+            (insert_dml, insert_params, insert_param_types),
+            update_dml,
+            delete_dml,
+        ]
+
+        with self.assertRaises(RuntimeError):
+            transaction.batch_update(dml_statements)
+
+        self.assertEqual(transaction._execute_sql_count, 1)
+
     def test_context_mgr_success(self):
         import datetime
         from google.cloud.spanner_v1.proto.spanner_pb2 import CommitResponse