Repository: googleapis/python-spanner-django

Issue:
dbapi: Cursor.execute should detect DML operations and run them in a batch transaction instead of single use transactions
Currently doing

```python
cur.execute("INSERT INTO Singers (SingerId, First, Last) VALUES (10, 'Jim', 'Jefferson')"
```
produces the error
```shell
Traceback (most recent call last):
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 79, in next
    return six.next(self._wrapped)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/grpc/_channel.py", line 364, in __next__
    return self._next()
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/grpc/_channel.py", line 358, in _next
    raise self
grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with:
	status = StatusCode.INVALID_ARGUMENT
	details = "DML statements may not be performed in single-use transactions, to avoid replay."
	debug_error_string = "{"created":"@1572997808.997042000","description":"Error received from peer ipv4:172.217.5.202:443","file":"src/core/lib/surface/call.cc","file_line":1046,"grpc_message":"DML statements may not be performed in single-use transactions, to avoid replay.","grpc_status":3}"
>

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "conn.py", line 95, in <module>
    main()
  File "conn.py", line 87, in main
    for row in cur:
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/google/cloud/spanner_v1/streamed.py", line 143, in __iter__
    self._consume_next()
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/google/cloud/spanner_v1/streamed.py", line 116, in _consume_next
    response = six.next(self._response_iterator)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/google/cloud/spanner_v1/snapshot.py", line 45, in _restart_on_unavailable
    for item in iterator:
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 81, in next
    six.raise_from(exceptions.from_grpc_error(exc), exc)
  File "<string>", line 3, in raise_from
google.api_core.exceptions.InvalidArgument: 400 DML statements may not be performed in single-use transactions, to avoid replay.
```

and for clarity the actual error is an InvalidArgument gRPC error
```shell
google.api_core.exceptions.InvalidArgument: 400 DML statements may not be performed in single-use transactions, to avoid replay.
```


Discussion:
The problem here comes from the Spanner Python client not being able to `execute_sql` with update/insert statements and we should instead invoke transaction.execute_update after parsing and recognizing the SQL, the Python client didn't provide a warning or any failures except proxying them to Cloud Spanner's backend, sigh! 

Resolving patch:
diff --git a/spanner/dbapi/connection.py b/spanner/dbapi/connection.py
--- a/spanner/dbapi/connection.py
+++ b/spanner/dbapi/connection.py
@@ -36,11 +36,11 @@ def close(self):
 
 
     def __enter__(self):
-        pass
+        return self
 
 
     def __exit__(self, etype, value, traceback):
-        pass
+        return self.close()
 
 
     def commit(self):
diff --git a/spanner/dbapi/cursor.py b/spanner/dbapi/cursor.py
--- a/spanner/dbapi/cursor.py
+++ b/spanner/dbapi/cursor.py
@@ -12,20 +12,34 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import google.api_core.exceptions as grpc_exceptions
+
 from .exceptions import (
+        Error,
+        IntegrityError,
+        OperationalError,
         ProgrammingError
 )
 
+from .parse_utils import (
+        classify_stmt,
+        STMT_NON_UPDATING
+)
+
+
+_UNSET_COUNT = -1
+
 class Cursor(object):
     def __init__(self, session):
         self.__session = session
         self.__itr = None
         self.__res = None
+        self.__row_count = _UNSET_COUNT
 
 
     @property
     def description(self):
-        if not self.__res.metadata:
+        if not (self.res and self.__res.metadata):
             return None
 
         row_type = self.__res.metadata.row_type
@@ -37,7 +51,7 @@ def description(self):
 
     @property
     def rowcount(self):
-        raise ProgrammingError('Unimplemented')
+        return self.__row_count
 
 
     def close(self):
@@ -66,16 +80,51 @@ def execute(self, sql, *args, **kwargs):
         if not self.__session:
             raise ProgrammingError('Cursor is not connected to the database')
 
-        # Immediately using:
-        #   iter(response)
-        # here, because this Spanner API doesn't provide
-        # easy mechanisms to detect when only a single item
-        # is returned or many, yet mixing results that
-        # are for .fetchone() with those that would result in
-        # many items returns a RuntimeError if .fetchone() is
-        # invoked and vice versa.
-        self.__res = self.__session.execute_sql(sql, *args, **kwargs)
-        self.__itr = iter(self.__res)
+        # Classify whether this is a read-only SQL statement.
+        try:
+            if classify_stmt(sql) == STMT_NON_UPDATING:
+                self.__do_execute_non_update(sql, *args, **kwargs)
+            else:
+                self.__session.run_in_transaction(self.__do_execute_update, sql, *args, **kwargs)
+
+        except grpc_exceptions.AlreadyExists as e:
+            raise IntegrityError(e.details if hasattr(e, 'details') else e)
+
+        except grpc_exceptions.InvalidArgument as e:
+            raise ProgrammingError(e.details if hasattr(e, 'details') else e)
+
+        except grpc_exceptions.InternalServerError as e:
+            raise OperationalError(e.details if hasattr(e, 'details') else e)
+
+        except as e: # Catch all other exceptions and re-raise them.
+            raise Error(e.details if hasattr(e, 'details') else e)
+
+
+    def __do_execute_update(self, transaction, sql, *args, **kwargs):
+        res = transaction.execute_update(sql, *args, **kwargs)
+        self.__itr = None
+        if type(res) == int:
+            self.__row_count = res
+
+        return res
+
+
+    def __do_execute_non_update(self, sql, *args, **kwargs):
+        res = self.__session.execute_sql(sql, *args, **kwargs)
+        if type(res) == int:
+            self.__row_count = res
+            self.__itr = None
+        else:
+            # Immediately using:
+            #   iter(response)
+            # here, because this Spanner API doesn't provide
+            # easy mechanisms to detect when only a single item
+            # is returned or many, yet mixing results that
+            # are for .fetchone() with those that would result in
+            # many items returns a RuntimeError if .fetchone() is
+            # invoked and vice versa.
+            self.__itr = iter(self.__res)
+            self.__row_count = _UNSET_COUNT
 
 
     def __enter__(self):
@@ -94,10 +143,16 @@ def executemany(self, operation, seq_of_params):
 
 
     def __next__(self):
+        if self.__itr == None:
+            raise ProgrammingError('no results to return')
+
         return next(self.__itr)
 
 
     def __iter__(self):
+        if self.__itr == None:
+            raise ProgrammingError('no results to return')
+
         return self.__itr
 
 
@@ -106,7 +161,7 @@ def fetchone(self):
 
 
     def fetchall(self):
-        return list(self.__itr)
+        return list(self.__iter__())
 
 
     @property
diff --git a/spanner/dbapi/parse_utils.py b/spanner/dbapi/parse_utils.py
--- a/spanner/dbapi/parse_utils.py
+++ b/spanner/dbapi/parse_utils.py
@@ -148,3 +148,16 @@ def validate_instance_config(instance_config):
                         instance_config, reINSTANCE_CONFIG.pattern)
 
     return None
+
+
+STMT_NON_UPDATING = 'NON_UPDATING'
+STMT_UPDATING = 'UPDATING'
+
+# Heuristic for identifying statements that don't need to be run as updates.
+re_NON_UPDATE = re.compile('^\s*(SELECT|ANALYZE|AUDIT|EXPLAIN|SHOW)', re.UNICODE|re.IGNORECASE)
+
+def classify_stmt(sql):
+    if re_NON_UPDATE.match(sql):
+        return STMT_NON_UPDATING
+
+    return STMT_UPDATING


Test patch: